Merge branch 'master' into feature/seq_no

* master: (516 commits)
  Avoid angering Log4j in TransportNodesActionTests
  Add trace logging when aquiring and releasing operation locks for replication requests
  Fix handler name on message not fully read
  Remove accidental import.
  Improve log message in TransportNodesAction
  Clean up of Script.
  Update Joda Time to version 2.9.5 (#21468)
  Remove unused ClusterService dependency from SearchPhaseController (#21421)
  Remove max_local_storage_nodes from elasticsearch.yml (#21467)
  Wait for all reindex subtasks before rethrottling
  Correcting a typo-Maan to Man-in README.textile (#21466)
  Fix InternalSearchHit#hasSource to return the proper boolean value (#21441)
  Replace all index date-math examples with the URI encoded form
  Fix typos (#21456)
  Adapt ES_JVM_OPTIONS packaging test to ubuntu-1204
  Add null check in InternalSearchHit#sourceRef to prevent NPE (#21431)
  Add VirtualBox version check (#21370)
  Export ES_JVM_OPTIONS for SysV init
  Skip reindex rethrottle tests with workers
  Make forbidden APIs be quieter about classpath warnings (#21443)
  ...

Author: jasontedor

.gitignore

@@ -33,6 +33,7 @@ dependency-reduced-pom.xml
 # testing stuff
 **/.local*
 .vagrant/
+/logs/
 
 # osx stuff
 .DS_Store

CONTRIBUTING.md

@@ -88,7 +88,8 @@ Contributing to the Elasticsearch codebase
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
 Make sure you have [Gradle](http://gradle.org) installed, as
-Elasticsearch uses it as its build system.
+Elasticsearch uses it as its build system. Gradle must be version 2.13 _exactly_ in
+order to build successfully.
 
 Eclipse users can automatically configure their IDE: `gradle eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the

README.textile

@@ -123,7 +123,7 @@ There are many more options to perform search, after all, it's a search product
 
 h3. Multi Tenant - Indices and Types
 
-Maan, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data.
+Man, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data.
 
 Elasticsearch supports multiple indices, as well as multiple types per index. In the previous example we used an index called @twitter@, with two types, @user@ and @tweet@.
 
@@ -200,7 +200,7 @@ We have just covered a very small portion of what Elasticsearch is all about. Fo
 
 h3. Building from Source
 
-Elasticsearch uses "Gradle":https://gradle.org for its build system. You'll need to have a modern version of Gradle installed - 2.13 should do.
+Elasticsearch uses "Gradle":https://gradle.org for its build system. You'll need to have version 2.13 of Gradle installed.
 
 In order to create a distribution, simply run the @gradle assemble@ command in the cloned directory.
 

TESTING.asciidoc

@@ -16,22 +16,6 @@ following:
 gradle assemble
 -----------------------------
 
-== Other test options
-
-To disable and enable network transport, set the `tests.es.node.mode` system property.
-
-Use network transport:
-
-------------------------------------
--Dtests.es.node.mode=network
-------------------------------------
-
-Use local transport (default since 1.3):
-
--------------------------------------
--Dtests.es.node.mode=local
--------------------------------------
-
 === Running Elasticsearch from a checkout
 
 In order to run Elasticsearch from source without building a package, you can
@@ -41,6 +25,12 @@ run it using Gradle:
 gradle run
 -------------------------------------
 
+or to attach a remote debugger, run it as:
+
+-------------------------------------
+gradle run --debug-jvm
+-------------------------------------
+
 === Test case filtering.
 
 - `tests.class` is a class-filtering shell-like glob pattern,
@@ -363,7 +353,6 @@ These are the linux flavors the Vagrantfile currently supports:
 
 * ubuntu-1204 aka precise
 * ubuntu-1404 aka trusty
-* ubuntu-1504 aka vivid
 * ubuntu-1604 aka xenial
 * debian-8 aka jessie, the current debian stable distribution
 * centos-6

Vagrantfile

@@ -30,13 +30,6 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/ubuntu-14.04-x86_64"
     ubuntu_common config
   end
-  config.vm.define "ubuntu-1504" do |config|
-    config.vm.box = "elastic/ubuntu-15.04-x86_64"
-    ubuntu_common config, extra: <<-SHELL
-      # Install Jayatana so we can work around it being present.
-      [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
-    SHELL
-  end
   config.vm.define "ubuntu-1604" do |config|
     config.vm.box = "elastic/ubuntu-16.04-x86_64"
     ubuntu_common config, extra: <<-SHELL
@@ -156,6 +149,7 @@ def dnf_common(config)
     update_command: "dnf check-update",
     update_tracking_file: "/var/cache/dnf/last_update",
     install_command: "dnf install -y",
+    install_command_retries: 5,
     java_package: "java-1.8.0-openjdk-devel")
   if Vagrant.has_plugin?("vagrant-cachier")
     # Autodetect doesn't work....
@@ -205,6 +199,7 @@ def provision(config,
     update_command: 'required',
     update_tracking_file: 'required',
     install_command: 'required',
+    install_command_retries: 0,
     java_package: 'required',
     extra: '')
   # Vagrant run ruby 2.0.0 which doesn't have required named parameters....
@@ -215,9 +210,27 @@ def provision(config,
   config.vm.provision "bats dependencies", type: "shell", inline: <<-SHELL
     set -e
     set -o pipefail
+
+    # Retry install command up to $2 times, if failed
+    retry_installcommand() {
+      n=0
+      while true; do
+        #{install_command} $1 && break
+        let n=n+1
+        if [ $n -ge $2 ]; then
+          echo "==> Exhausted retries to install $1"
+          return 1
+        fi
+        echo "==> Retrying installing $1, attempt $((n+1))"
+        # Add a small delay to increase chance of metalink providing updated list of mirrors
+        sleep 5
+      done
+    }
+
     installed() {
       command -v $1 2>&1 >/dev/null
     }
+
     install() {
       # Only apt-get update if we haven't in the last day
       if [ ! -f #{update_tracking_file} ] || [ "x$(find #{update_tracking_file} -mtime +0)" == "x#{update_tracking_file}" ]; then
@@ -226,8 +239,14 @@ def provision(config,
         touch #{update_tracking_file}
       fi
       echo "==> Installing $1"
-      #{install_command} $1
+      if [ #{install_command_retries} -eq 0 ]
+      then
+            #{install_command} $1
+      else
+            retry_installcommand $1 #{install_command_retries}
+      fi
     }
+
     ensure() {
       installed $1 || install $1
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -58,57 +58,57 @@ public class AllocationBenchmark {
     // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would
     // need its own main method and we cannot execute more than one class with a main method per JAR.
     @Param({
-        // indices, shards, replicas, nodes
-        "       10,      1,        0,     1",
-        "       10,      3,        0,     1",
-        "       10,     10,        0,     1",
-        "      100,      1,        0,     1",
-        "      100,      3,        0,     1",
-        "      100,     10,        0,     1",
-
-        "       10,      1,        0,    10",
-        "       10,      3,        0,    10",
-        "       10,     10,        0,    10",
-        "      100,      1,        0,    10",
-        "      100,      3,        0,    10",
-        "      100,     10,        0,    10",
-
-        "       10,      1,        1,    10",
-        "       10,      3,        1,    10",
-        "       10,     10,        1,    10",
-        "      100,      1,        1,    10",
-        "      100,      3,        1,    10",
-        "      100,     10,        1,    10",
-
-        "       10,      1,        2,    10",
-        "       10,      3,        2,    10",
-        "       10,     10,        2,    10",
-        "      100,      1,        2,    10",
-        "      100,      3,        2,    10",
-        "      100,     10,        2,    10",
-
-        "       10,      1,        0,    50",
-        "       10,      3,        0,    50",
-        "       10,     10,        0,    50",
-        "      100,      1,        0,    50",
-        "      100,      3,        0,    50",
-        "      100,     10,        0,    50",
-
-        "       10,      1,        1,    50",
-        "       10,      3,        1,    50",
-        "       10,     10,        1,    50",
-        "      100,      1,        1,    50",
-        "      100,      3,        1,    50",
-        "      100,     10,        1,    50",
-
-        "       10,      1,        2,    50",
-        "       10,      3,        2,    50",
-        "       10,     10,        2,    50",
-        "      100,      1,        2,    50",
-        "      100,      3,        2,    50",
-        "      100,     10,        2,    50"
+        // indices| shards| replicas| nodes
+        "       10|      1|        0|     1",
+        "       10|      3|        0|     1",
+        "       10|     10|        0|     1",
+        "      100|      1|        0|     1",
+        "      100|      3|        0|     1",
+        "      100|     10|        0|     1",
+
+        "       10|      1|        0|    10",
+        "       10|      3|        0|    10",
+        "       10|     10|        0|    10",
+        "      100|      1|        0|    10",
+        "      100|      3|        0|    10",
+        "      100|     10|        0|    10",
+
+        "       10|      1|        1|    10",
+        "       10|      3|        1|    10",
+        "       10|     10|        1|    10",
+        "      100|      1|        1|    10",
+        "      100|      3|        1|    10",
+        "      100|     10|        1|    10",
+
+        "       10|      1|        2|    10",
+        "       10|      3|        2|    10",
+        "       10|     10|        2|    10",
+        "      100|      1|        2|    10",
+        "      100|      3|        2|    10",
+        "      100|     10|        2|    10",
+
+        "       10|      1|        0|    50",
+        "       10|      3|        0|    50",
+        "       10|     10|        0|    50",
+        "      100|      1|        0|    50",
+        "      100|      3|        0|    50",
+        "      100|     10|        0|    50",
+
+        "       10|      1|        1|    50",
+        "       10|      3|        1|    50",
+        "       10|     10|        1|    50",
+        "      100|      1|        1|    50",
+        "      100|      3|        1|    50",
+        "      100|     10|        1|    50",
+
+        "       10|      1|        2|    50",
+        "       10|      3|        2|    50",
+        "       10|     10|        2|    50",
+        "      100|      1|        2|    50",
+        "      100|      3|        2|    50",
+        "      100|     10|        2|    50"
     })
-    public String indicesShardsReplicasNodes = "10,1,0,1";
+    public String indicesShardsReplicasNodes = "10|1|0|1";
 
     public int numTags = 2;
 
@@ -117,7 +117,7 @@ public class AllocationBenchmark {
 
     @Setup
     public void setUp() throws Exception {
-        final String[] params = indicesShardsReplicasNodes.split(",");
+        final String[] params = indicesShardsReplicasNodes.split("\\|");
 
         int numIndices = toInt(params[0]);
         int numShards = toInt(params[1]);

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java

@@ -31,15 +31,18 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 
 import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public final class Allocators {
     private static class NoopGatewayAllocator extends GatewayAllocator {
@@ -91,8 +94,11 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl
 
     }
 
+    private static final AtomicInteger portGenerator = new AtomicInteger();
+
     public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
-        return new DiscoveryNode("", nodeId, LocalTransportAddress.buildUnique(), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
+        return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS,
+            portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
             DiscoveryNode.Role.DATA), Version.CURRENT);
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -28,6 +28,7 @@ import org.gradle.api.Task
 import org.gradle.api.XmlProvider
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.ModuleDependency
+import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
@@ -294,12 +295,15 @@ class BuildPlugin implements Plugin<Project> {
      * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
      *
      * <ul>
-     *     <li>Remove transitive dependencies (using wildcard exclusions, fixed in gradle 2.14)</li>
-     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)
+     *     <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
+     *         as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
+     *         being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
+     *         translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
+     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
      * </ul>
      */
     private static Closure fixupDependencies(Project project) {
-        // TODO: remove this when enforcing gradle 2.14+, it now properly handles exclusions
+        // TODO: revisit this when upgrading to Gradle 2.14+, see Javadoc comment above
         return { XmlProvider xml ->
             // first find if we have dependencies at all, and grab the node
             NodeList depsNodes = xml.asNode().get('dependencies')
@@ -334,10 +338,19 @@ class BuildPlugin implements Plugin<Project> {
                     continue
                 }
 
-                // we now know we have something to exclude, so add a wildcard exclusion element
-                Node exclusion = depNode.appendNode('exclusions').appendNode('exclusion')
-                exclusion.appendNode('groupId', '*')
-                exclusion.appendNode('artifactId', '*')
+                // we now know we have something to exclude, so add exclusions for all artifacts except the main one
+                Node exclusions = depNode.appendNode('exclusions')
+                for (ResolvedArtifact artifact : artifacts) {
+                    ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
+                    String depGroupId = moduleVersionIdentifier.group
+                    String depArtifactId = moduleVersionIdentifier.name
+                    // add exclusions for all artifacts except the main one
+                    if (depGroupId != groupId || depArtifactId != artifactId) {
+                        Node exclusion = exclusions.appendNode('exclusion')
+                        exclusion.appendNode('groupId', depGroupId)
+                        exclusion.appendNode('artifactId', depArtifactId)
+                    }
+                }
             }
         }
     }
@@ -393,7 +406,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 options.encoding = 'UTF-8'
-                //options.incremental = true
+                options.incremental = true
 
                 if (project.javaVersion == JavaVersion.VERSION_1_9) {
                     // hack until gradle supports java 9's new "--release" arg

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -38,7 +38,7 @@ public class DocsTestPlugin extends RestTestPlugin {
              * the last released version for docs. */
             '\\{version\\}':
                 VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
-            '\\{lucene_version\\}' : VersionProperties.lucene,
+            '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'