Back port build changes from elastic#39102

This back-ports how versions are determined and bwc test are set up from
 elastic#39102 without enabling the bwc from current version tests so it's
 easier/possible to backmerge future buld changes.
It's expected that the tets are lacking many of the required fixes in
this version to enable them.

Author: alpar-t

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -216,7 +216,8 @@ class ClusterFormationTasks {
         }
         if (unreleasedInfo != null) {
             dependency = project.dependencies.project(
-                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+                    path: unreleasedInfo.gradleProjectPath, configuration: snapshotProject
+            )
         } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
             dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
         } else {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -199,10 +199,10 @@ class VagrantTestPlugin implements Plugin<Project> {
                 // handle snapshots pointing to bwc build
                 UPGRADE_FROM_ARCHIVES.each {
                     dependencies.add(project.dependencies.project(
-                            path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
+                            path: "${unreleasedInfo.gradleProjectPath}", configuration: it))
                     if (upgradeFromVersion.onOrAfter('6.3.0')) {
                         dependencies.add(project.dependencies.project(
-                                path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
+                                path: "${unreleasedInfo.gradleProjectPath}", configuration: "oss-${it}"))
                     }
                 }
             } else {

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -57,7 +57,7 @@
  * <ul>
  * <li>the unreleased <b>major</b>, M+1.0.0 on the `master` branch</li>
  * <li>the unreleased <b>minor</b>,  M.N.0 on the `M.x` (x is literal) branch</li>
- * <li>the unreleased <b>bugfix</b>, M.N.c (c &gt; 0) on the `M.b` branch</li>
+ * <li>the unreleased <b>bugfix</b>, M.N.c (c &gt; 0) on the `M.N` branch</li>
  * <li>the unreleased <b>maintenance</b>, M-1.d.e ( d &gt; 0, e &gt; 0) on the `(M-1).d` branch</li>
  * </ul>
  * In addition to these, there will be a fifth one when a minor reaches feature freeze, we call this the <i>staged</i>
@@ -74,7 +74,7 @@
  * We can reliably figure out which the unreleased versions are due to the convention of always adding the next unreleased
  * version number to server in all branches when a version is released.
  * E.x when M.N.c is released M.N.c+1 is added to the Version class mentioned above in all the following branches:
- *  `M.b`, `M.x` and `master` so we can reliably assume that the leafs of the version tree are unreleased.
+ *  `M.N`, `M.x` and `master` so we can reliably assume that the leafs of the version tree are unreleased.
  * This convention is enforced by checking the versions we consider to be unreleased against an
  * authoritative source (maven central).
  * We are then able to map the unreleased version to branches in git and Gradle projects that are capable of checking
@@ -93,12 +93,12 @@ public class VersionCollection {
     public class UnreleasedVersionInfo {
         public final Version version;
         public final String branch;
-        public final String gradleProjectName;
+        public final String gradleProjectPath;
 
-        UnreleasedVersionInfo(Version version, String branch, String gradleProjectName) {
+        UnreleasedVersionInfo(Version version, String branch, String gradleProjectPath) {
             this.version = version;
             this.branch = branch;
-            this.gradleProjectName = gradleProjectName;
+            this.gradleProjectPath = gradleProjectPath;
         }
     }
 
@@ -135,11 +135,8 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
 
         Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
         for (Version unreleasedVersion : getUnreleased()) {
-            if (unreleasedVersion.equals(currentVersion)) {
-                continue;
-            }
             unreleased.put(unreleasedVersion,
-                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
+                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectPathFor(unreleasedVersion)));
         }
         this.unreleased = Collections.unmodifiableMap(unreleased);
     }
@@ -176,17 +173,19 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
             .map(version -> new UnreleasedVersionInfo(
                     version,
                     getBranchFor(version),
-                    getGradleProjectNameFor(version)
+                    getGradleProjectPathFor(version)
                 )
             )
             .collect(Collectors.toList());
 
         collect.forEach(uvi -> consumer.accept(uvi));
     }
 
-    private String getGradleProjectNameFor(Version version) {
+    private String getGradleProjectPathFor(Version version) {
+        // We have Gradle projects set up to check out and build unreleased versions based on the our branching
+        // conventions described in this classes javadoc
         if (version.equals(currentVersion)) {
-            throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build.");
+            return ":distribution";
         }
 
         Map<Integer, List<Version>> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor();
@@ -197,27 +196,33 @@ private String getGradleProjectNameFor(Version version) {
                 .collect(Collectors.toList());
             if (unreleasedStagedOrMinor.size() > 2) {
                 if (unreleasedStagedOrMinor.get(unreleasedStagedOrMinor.size() - 2).equals(version)) {
-                    return "minor";
+                    return ":distribution:bwc:minor";
                 } else{
-                    return "staged";
+                    return ":distribution:bwc:staged";
                 }
             } else {
-                return "minor";
+                return ":distribution:bwc:minor";
             }
         } else {
             if (releasedMajorGroupedByMinor
                 .getOrDefault(version.getMinor(), emptyList())
                 .contains(version)) {
-                return "bugfix";
+                return ":distribution:bwc:bugfix";
             } else {
-                return "maintenance";
+                return ":distribution:bwc:maintenance";
             }
         }
     }
 
     private String getBranchFor(Version version) {
-        switch (getGradleProjectNameFor(version)) {
-            case "minor":
+        // based on the rules described in this classes javadoc, figure out the branch on which an unreleased version
+        // lives.
+        // We do this based on the Gradle project path because there's a direct correlation, so we dont have to duplicate
+        // the logic from there
+        switch (getGradleProjectPathFor(version)) {
+            case ":distribution":
+                return "master";
+            case ":distribution:bwc:minor":
                 // The .x branch will always point to the latest minor (for that major), so a "minor" project will be on the .x branch
                 //  unless there is more recent (higher) minor.
                 final Version latestInMajor = getLatestVersionByKey(groupByMajor, version.getMajor());
@@ -226,9 +231,9 @@ private String getBranchFor(Version version) {
                 } else {
                     return version.getMajor() + "." + version.getMinor();
                 }
-            case "staged":
-            case "maintenance":
-            case "bugfix":
+            case ":distribution:bwc:staged":
+            case ":distribution:bwc:maintenance":
+            case ":distribution:bwc:bugfix":
                 return version.getMajor() + "." + version.getMinor();
             default:
                 throw new IllegalStateException("Unexpected Gradle project name");
@@ -237,6 +242,7 @@ private String getBranchFor(Version version) {
 
     public List<Version> getUnreleased() {
         List<Version> unreleased = new ArrayList<>();
+
         // The current version is being worked, is always unreleased
         unreleased.add(currentVersion);
 
@@ -345,6 +351,7 @@ public List<Version> getIndexCompatible() {
                 .filter(version -> version.equals(currentVersion) == false)
                 .collect(Collectors.toList())
         );
+
     }
 
     public List<Version> getWireCompatible() {

buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java

@@ -110,24 +110,24 @@ public void testExceptionOnTooManyMajors() {
 
     public void testWireCompatible() {
         assertVersionsEquals(
-            singletonList("6.5.0-SNAPSHOT"),
+            asList("6.5.0"),
             getVersionCollection("7.0.0-alpha1").getWireCompatible()
         );
         assertVersionsEquals(
             asList(
                 "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
-                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "5.6.11", "5.6.12", "5.6.13",
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
                 "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
-                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT"
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2"
             ),
             getVersionCollection("6.5.0").getWireCompatible()
         );
 
         assertVersionsEquals(
             asList(
                 "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
-                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT", "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
+                "5.6.11", "5.6.12", "5.6.13", "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
                 "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4", "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1"
             ),
             getVersionCollection("6.4.2").getWireCompatible()
@@ -136,16 +136,16 @@ public void testWireCompatible() {
         assertVersionsEquals(
             asList(
                 "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
-                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "5.6.11", "5.6.12", "5.6.13",
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
                 "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
-                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2", "6.5.0"
             ),
             getVersionCollection("6.6.0").getWireCompatible()
         );
 
         assertVersionsEquals(
-            singletonList("7.3.0"),
+            asList("7.3.0"),
             getVersionCollection("8.0.0").getWireCompatible()
         );
         assertVersionsEquals(
@@ -157,26 +157,26 @@ public void testWireCompatible() {
 
     public void testWireCompatibleUnreleased() {
         assertVersionsEquals(
-            singletonList("6.5.0-SNAPSHOT"),
+            asList("6.5.0"),
             getVersionCollection("7.0.0-alpha1").getUnreleasedWireCompatible()
         );
         assertVersionsEquals(
-            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT"),
+            asList("5.6.13", "6.4.2"),
             getVersionCollection("6.5.0").getUnreleasedWireCompatible()
         );
 
         assertVersionsEquals(
-            singletonList("5.6.13-SNAPSHOT"),
+            asList("5.6.13"),
             getVersionCollection("6.4.2").getUnreleasedWireCompatible()
         );
 
         assertVersionsEquals(
-            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            asList("5.6.13", "6.4.2", "6.5.0"),
             getVersionCollection("6.6.0").getUnreleasedWireCompatible()
         );
 
         assertVersionsEquals(
-            singletonList("7.3.0"),
+            asList("7.3.0"),
             getVersionCollection("8.0.0").getUnreleasedWireCompatible()
         );
         assertVersionsEquals(
@@ -190,7 +190,7 @@ public void testIndexCompatible() {
             asList(
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
                 "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4", "6.3.0", "6.3.1",
-                "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+                "6.3.2", "6.4.0", "6.4.1", "6.4.2", "6.5.0"
             ),
             getVersionCollection("7.0.0-alpha1").getIndexCompatible()
         );
@@ -199,9 +199,9 @@ public void testIndexCompatible() {
             asList(
                 "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
                 "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
-                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13",
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
-                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT"
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2"
             ),
             getVersionCollection("6.5.0").getIndexCompatible()
         );
@@ -210,7 +210,7 @@ public void testIndexCompatible() {
             asList(
                 "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
                 "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
-                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13",
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
                 "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1"
             ),
@@ -221,9 +221,9 @@ public void testIndexCompatible() {
             asList(
                 "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
                 "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
-                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13",
                 "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
-                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2", "6.5.0"
             ),
             getVersionCollection("6.6.0").getIndexCompatible()
         );
@@ -236,22 +236,22 @@ public void testIndexCompatible() {
 
     public void testIndexCompatibleUnreleased() {
         assertVersionsEquals(
-            asList("6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            asList("6.4.2", "6.5.0"),
             getVersionCollection("7.0.0-alpha1").getUnreleasedIndexCompatible()
         );
 
         assertVersionsEquals(
-            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT"),
+            asList("5.6.13", "6.4.2"),
             getVersionCollection("6.5.0").getUnreleasedIndexCompatible()
         );
 
         assertVersionsEquals(
-            singletonList("5.6.13-SNAPSHOT"),
+            asList("5.6.13"),
             getVersionCollection("6.4.2").getUnreleasedIndexCompatible()
         );
 
         assertVersionsEquals(
-            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            asList("5.6.13", "6.4.2", "6.5.0"),
             getVersionCollection("6.6.0").getUnreleasedIndexCompatible()
         );
 
@@ -315,29 +315,29 @@ public void testGetBranch() {
         );
     }
 
-    public void testGetGradleProjectName() {
-        assertUnreleasedGradleProjectNames(
-            asList("bugfix", "minor"),
+    public void testGetGradleProjectPath() {
+        assertUnreleasedGradleProjectPaths(
+            asList(":distribution:bwc:bugfix", ":distribution:bwc:minor"),
             getVersionCollection("7.0.0-alpha1")
         );
-        assertUnreleasedGradleProjectNames(
-            asList("maintenance", "bugfix"),
+        assertUnreleasedGradleProjectPaths(
+            asList(":distribution:bwc:maintenance", ":distribution:bwc:bugfix"),
             getVersionCollection("6.5.0")
         );
-        assertUnreleasedGradleProjectNames(
-            singletonList("maintenance"),
+        assertUnreleasedGradleProjectPaths(
+            singletonList(":distribution:bwc:maintenance"),
             getVersionCollection("6.4.2")
         );
-        assertUnreleasedGradleProjectNames(
-            asList("maintenance", "bugfix", "minor"),
+        assertUnreleasedGradleProjectPaths(
+            asList(":distribution:bwc:maintenance", ":distribution:bwc:bugfix", ":distribution:bwc:minor"),
             getVersionCollection("6.6.0")
         );
-        assertUnreleasedGradleProjectNames(
-            asList("bugfix", "staged", "minor"),
+        assertUnreleasedGradleProjectPaths(
+            asList(":distribution:bwc:bugfix", ":distribution:bwc:staged", ":distribution:bwc:minor"),
             getVersionCollection("8.0.0")
         );
-        assertUnreleasedGradleProjectNames(
-            asList("maintenance", "staged", "minor"),
+        assertUnreleasedGradleProjectPaths(
+            asList(":distribution:bwc:maintenance", ":distribution:bwc:staged", ":distribution:bwc:minor"),
             getVersionCollection("7.1.0")
         );
     }
@@ -390,10 +390,10 @@ public void testCompareToAuthoritativeNotReallyRelesed() {
         vc.compareToAuthoritative(authoritativeReleasedVersions);
     }
 
-    private void assertUnreleasedGradleProjectNames(List<String> expectedNAmes, VersionCollection versionCollection) {
+    private void assertUnreleasedGradleProjectPaths(List<String> expectedNAmes, VersionCollection versionCollection) {
         List<String> actualNames = new ArrayList<>();
         versionCollection.forPreviousUnreleased(unreleasedVersion ->
-            actualNames.add(unreleasedVersion.gradleProjectName)
+            actualNames.add(unreleasedVersion.gradleProjectPath)
         );
         assertEquals(expectedNAmes, actualNames);
     }

distribution/build.gradle

@@ -516,3 +516,16 @@ subprojects {
     }
   }
 }
+
+['archives:windows-zip','archives:oss-windows-zip',
+ 'archives:darwin-tar','archives:oss-darwin-tar',
+ 'archives:linux-tar', 'archives:oss-linux-tar',
+ 'packages:rpm', 'packages:deb',
+ 'packages:oss-rpm', 'packages:oss-deb',
+].forEach { subName ->
+  Project subproject = project("${project.path}:${subName}")
+  Configuration configuration = configurations.create(subproject.name)
+  dependencies {
+    "${configuration.name}" project(subproject.path)
+  }
+}

distribution/bwc/build.gradle

@@ -33,7 +33,7 @@ import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
  *  unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
  *  and configure them to build various versions here.
  */
-bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> project("${project.path}:${unreleasedVersion.gradleProjectName}") {
+bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> project("${unreleasedVersion.gradleProjectPath}") {
     Version bwcVersion = unreleasedVersion.version
     String bwcBranch = unreleasedVersion.branch
     apply plugin: 'distribution'