From 91bf2c97de1024501a127e71f7c7adfc544bfba9 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 21 Jan 2020 18:43:48 +0900 Subject: [PATCH 1/4] Add a fallback repository as GCS --- .github/workflows/master.yml | 8 -------- pom.xml | 19 ++++++++++++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e4d2d8470cc34..d53119ad75599 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -66,14 +66,6 @@ jobs: export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" mkdir -p ~/.m2 - # `Maven Central` is too flaky in terms of downloading artifacts in `GitHub Action` environment. - # `Google Maven Central Mirror` is too slow in terms of sycing upstream. To get the best combination, - # 1) we set `Google Maven Central` as a mirror of `central` in `GitHub Action` environment only. - # 2) we duplicates `Maven Central` in pom.xml with ID `central_without_mirror`. - # In other words, in GitHub Action environment, `central` is mirrored by `Google Maven Central` first. - # If `Google Maven Central` doesn't provide the artifact due to its slowness, `central_without_mirror` will be used. - # Note that we aim to achieve the above while keeping the existing behavior of non-`GitHub Action` environment unchanged. - echo "google-maven-centralGCS Maven Central mirrorhttps://maven-central.storage-download.googleapis.com/repos/central/data/central" > ~/.m2/settings.xml ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install rm -rf ~/.m2/repository/org/apache/spark diff --git a/pom.xml b/pom.xml index d2b2b0205060f..f50bae9122e5e 100644 --- a/pom.xml +++ b/pom.xml @@ -258,14 +258,12 @@ - central_without_mirror + google-maven-central - Maven Repository - https://repo.maven.apache.org/maven2 + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/repos/central/data true @@ -285,6 +283,17 @@ false + + google-maven-central + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/repos/central/data + + true + + + false + + GCS Maven Central mirror https://maven-central.storage-download.googleapis.com/repos/central/data From 556b88d78eb080debb4fb21e165bd2c73d1291bb Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Wed, 22 Jan 2020 09:41:08 +0900 Subject: [PATCH 3/4] switch the order --- pom.xml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index a426eab70955e..fb0baf5cf7a10 100644 --- a/pom.xml +++ b/pom.xml @@ -246,10 +246,10 @@ - central - - Maven Repository - https://repo.maven.apache.org/maven2 + google-maven-central + + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/repos/central/data true @@ -258,12 +258,12 @@ - google-maven-central - GCS Maven Central mirror - https://maven-central.storage-download.googleapis.com/repos/central/data + central + Maven Repository + https://repo.maven.apache.org/maven2 true @@ -274,8 +274,9 @@ - central - https://repo.maven.apache.org/maven2 + google-maven-central + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/repos/central/data true @@ -284,9 +285,8 @@ - google-maven-central - GCS Maven Central mirror - https://maven-central.storage-download.googleapis.com/repos/central/data + central + https://repo.maven.apache.org/maven2 true From 0b9d198b21fc07bce904207437ae14f9bb73c7d4 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Thu, 23 Jan 2020 12:36:28 +0900 Subject: [PATCH 4/4] Use consistently in SBT as well --- pom.xml | 17 ++++++++++++----- project/SparkBuild.scala | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index fb0baf5cf7a10..4e6c571ee052c 100644 --- a/pom.xml +++ b/pom.xml @@ -246,10 +246,13 @@ - google-maven-central - + gcs-maven-central-mirror + GCS Maven Central mirror - https://maven-central.storage-download.googleapis.com/repos/central/data + https://maven-central.storage-download.googleapis.com/repos/central/data/ true @@ -274,9 +277,13 @@ - google-maven-central + gcs-maven-central-mirror + GCS Maven Central mirror - https://maven-central.storage-download.googleapis.com/repos/central/data + https://maven-central.storage-download.googleapis.com/repos/central/data/ true diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9385505ad8ac5..87e584bd28b68 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -224,6 +224,9 @@ object SparkBuild extends PomBuild { // Override SBT's default resolvers: resolvers := Seq( + // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central. + // See https://storage-download.googleapis.com/maven-central/index.html for more info. + "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/repos/central/data/", DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)