Skip to content

build-tools check are fixed on aarch64 #68630

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 10, 2021
Merged

build-tools check are fixed on aarch64 #68630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
37ed04d
Igmore BwcVersionsTests on aarch64
breskeby Feb 7, 2021
ee800d7
Fix spotless
breskeby Feb 7, 2021
809291d
Introduce classifier field to Architecture
breskeby Feb 8, 2021
c07d16c
Fix more build tools tests on aarch64
breskeby Feb 8, 2021
ceba708
Fix InternalDistributionDownloadPluginFuncTest on aarch64
breskeby Feb 8, 2021
a2879c3
More aaarch build tools test fixes
breskeby Feb 8, 2021
cb89ee6
More fixes
breskeby Feb 8, 2021
a042870
Add test coverage for downloading adoptjdk aarch64 jdks
breskeby Feb 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ...c/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,19 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
when:
def guh = new File(testProjectDir.getRoot(), "gradle-user-home").absolutePath;
def runner = gradleRunner('clean', 'setupDistro', '-i', '-g', guh)
def unpackingMessage = "Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " +
"using SymbolicLinkPreservingUntarTransform"
def result = withMockedDistributionDownload(version, platform, runner) {
// initial run
def firstRun = build()
assertOutputContains(firstRun.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
"using SymbolicLinkPreservingUntarTransform")
assertOutputContains(firstRun.output, unpackingMessage)
// 2nd invocation
build()
}

then:
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
assertOutputMissing(result.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
"using SymbolicLinkPreservingUntarTransform")
assertOutputMissing(result.output, unpackingMessage)
}

def "transforms are reused across projects"() {
Expand Down Expand Up @@ -100,7 +100,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {

then:
result.tasks.size() == 3
result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
result.output.count("Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " +
"using SymbolicLinkPreservingUntarTransform") == 1
}

Expand Down
43 changes: 25 additions & 18 deletions buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,16 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {

private static final String OPENJDK_VERSION_OLD = "1+99"
private static final String ADOPT_JDK_VERSION = "12.0.2+10"
private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9"
private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
private static final String AZUL_AARCH_VERSION = "15.0.1+99@123456789123456789123456789abcde"
private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)");

@Unroll
def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
given:
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform);
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
def mockedContent = filebytes(jdkVendor, platform)
buildFile.text = """
plugins {
Expand Down Expand Up @@ -70,20 +72,22 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
assertExtraction(result.output, expectedJavaBin);

where:
platform | arch | jdkVendor | jdkVersion | expectedJavaBin | suffix
"linux" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)"
"windows" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)"
"darwin" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | ""
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
"darwin" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "Contents/Home/bin/java" | ""
"linux" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "bin/java" | ""
platform | arch | jdkVendor | jdkVersion | expectedJavaBin | suffix
"linux" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | ""
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)"
"windows" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | ""
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)"
"darwin" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | ""
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | ""
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
"darwin" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "Contents/Home/bin/java" | ""
"linux" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "bin/java" | ""
"linux" | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_11 | "bin/java" | "(jdk 11)"
"linux" | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_15 | "bin/java" | "(jdk 15)"
}

def "transforms are reused across projects"() {
Expand Down Expand Up @@ -195,10 +199,13 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
true
}

private static String urlPath(final String vendor, final String version, final String platform) {
private static String urlPath(final String vendor,
final String version,
final String platform,
final String arch = 'x64') {
if (vendor.equals(VENDOR_ADOPTOPENJDK)) {
final String module = isMac(platform) ? "mac" : platform;
return "/jdk-12.0.2+10/" + module + "/x64/jdk/hotspot/normal/adoptopenjdk";
return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptopenjdk";
} else if (vendor.equals(VENDOR_OPENJDK)) {
final String effectivePlatform = isMac(platform) ? "osx" : platform;
final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
Expand All @@ -208,7 +215,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
} else if (vendor.equals(VENDOR_AZUL)) {
final String module = isMac(platform) ? "macosx" : platform;
// we only test zulu 15 darwin aarch64 for now
return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_aarch64.tar.gz";
return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_${arch}.tar.gz";
}
}

Expand Down
3 changes: 1 addition & 2 deletions ...src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,7 @@ class DistributionDownloadFixture {
private static String urlPath(String version,ElasticsearchDistribution.Platform platform) {
String fileType = ((platform == ElasticsearchDistribution.Platform.LINUX ||
platform == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
String arch = Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64"
"/downloads/elasticsearch/elasticsearch-${version}-${platform}-${arch}.$fileType"
"/downloads/elasticsearch/elasticsearch-${version}-${platform}-${Architecture.current().classifier}.$fileType"
}

private static byte[] filebytes(String urlPath) throws IOException {
Expand Down
25 changes: 12 additions & 13 deletions ...roovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,11 @@

package org.elasticsearch.gradle.internal

import org.elasticsearch.gradle.Architecture
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
import org.gradle.testkit.runner.GradleRunner
import org.gradle.testkit.runner.TaskOutcome
import org.junit.Rule
import org.junit.rules.TemporaryFolder

import java.lang.management.ManagementFactory

class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {

Expand Down Expand Up @@ -61,7 +58,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()

then:
result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
result.task(":distribution:archives:${testArchiveProjectName}:buildExpanded").outcome == TaskOutcome.SUCCESS
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
}
Expand Down Expand Up @@ -133,24 +130,24 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
apply plugin:'base'

// packed distro
configurations.create("linux-tar")
configurations.create("${testArchiveProjectName}")
tasks.register("buildBwcTask", Tar) {
from('bwc-marker.txt')
archiveExtension = "tar.gz"
compression = Compression.GZIP
}
artifacts {
it.add("linux-tar", buildBwcTask)
it.add("${testArchiveProjectName}", buildBwcTask)
}

// expanded distro
configurations.create("expanded-linux-tar")
configurations.create("expanded-${testArchiveProjectName}")
def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
from('bwc-marker.txt')
into('build/install/elastic-distro')
}
artifacts {
it.add("expanded-linux-tar", file('build/install')) {
it.add("expanded-${testArchiveProjectName}", file('build/install')) {
builtBy expandedTask
type = 'directory'
}
Expand All @@ -160,9 +157,9 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest

private void localDistroSetup() {
settingsFile << """
include ":distribution:archives:linux-tar"
include ":distribution:archives:${testArchiveProjectName}"
"""
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", testArchiveProjectName)
new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
new File(bwcSubProjectFolder, 'build.gradle') << """
import org.gradle.api.internal.artifacts.ArtifactAttributes;
Expand Down Expand Up @@ -190,10 +187,12 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
it.add("extracted", buildExpanded)
}
"""
buildFile << """
"""
}

String getTestArchiveProjectName() {
def archSuffix = Architecture.current() == Architecture.AARCH64 ? '-aarch64' : ''
return "linux${archSuffix}-tar"
}
boolean assertExtractedDistroIsCreated(String relativeDistroPath, String markerFileName) {
File extractedFolder = new File(testProjectDir.root, relativeDistroPath)
assert extractedFolder.exists()
Expand Down
4 changes: 2 additions & 2 deletions .../resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@
# in compliance with, at your election, the Elastic License 2.0 or the Server
# Side Public License, v 1.
#
ES_BUILD_JAVA=openjdk12
ES_RUNTIME_JAVA=openjdk12
ES_BUILD_JAVA=openjdk11
ES_RUNTIME_JAVA=openjdk11
GRADLE_TASK=build
10 changes: 8 additions & 2 deletions buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,14 @@

public enum Architecture {

X64,
AARCH64;
X64("x86_64"),
AARCH64("aarch64");

public final String classifier;

Architecture(String classifier) {
this.classifier = classifier;
}

public static Architecture current() {
final String architecture = System.getProperty("os.arch", "");
Expand Down
7 changes: 2 additions & 5 deletions buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,11 @@ private String dependencyNotation(ElasticsearchDistribution distribution) {

Version distroVersion = Version.fromString(distribution.getVersion());
String extension = distribution.getType().toString();
String classifier = ":" + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
String classifier = ":" + Architecture.current().classifier;
if (distribution.getType() == Type.ARCHIVE) {
extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
if (distroVersion.onOrAfter("7.0.0")) {
classifier = ":"
+ distribution.getPlatform()
+ "-"
+ (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
classifier = ":" + distribution.getPlatform() + "-" + Architecture.current().classifier;
} else {
classifier = "";
}
Expand Down
7 changes: 7 additions & 0 deletions buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package org.elasticsearch.gradle;

import org.elasticsearch.gradle.test.GradleUnitTestCase;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
Expand Down Expand Up @@ -279,6 +281,11 @@ public class BwcVersionsTests extends GradleUnitTestCase {
sampleVersions.put("7.1.0", asList("7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0"));
}

@BeforeClass
public static void setupAll() {
Assume.assumeFalse(Architecture.current() == Architecture.AARCH64);
Copy link
Contributor Author

@breskeby breskeby Feb 9, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now we just ignore the unit tests on aarch64 as we filter out there anything < 7.12

}

@Test(expected = IllegalArgumentException.class)
public void testExceptionOnEmpty() {
new BwcVersions(asList("foo", "bar"), Version.fromString("7.0.0"));
Expand Down