From 517ff708d6591d02e85ac5e3723745c8626c9d07 Mon Sep 17 00:00:00 2001 From: Arjun Sharda <77706434+ArjunSharda@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:41:55 -0500 Subject: [PATCH 1/5] Update benchmark.yml --- .github/workflows/benchmark.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a322fe065b57..86418db17f79 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -75,12 +75,12 @@ jobs: SPARK_HOME: ${{ github.workspace }} steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to get diff files with: fetch-depth: 0 - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -91,14 +91,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: benchmark-coursier-${{ github.event.inputs.jdk }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | benchmark-coursier-${{ github.event.inputs.jdk }} - name: Install Java ${{ github.event.inputs.jdk }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ github.event.inputs.jdk }} - name: Run benchmarks @@ -121,7 +121,7 @@ jobs: echo "Preparing the benchmark results:" tar -cvf benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar `git diff --name-only` `git ls-files --others --exclude-standard` - name: Upload benchmark results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}-${{ matrix.split }} path: benchmark-results-${{ github.event.inputs.jdk }}-${{ github.event.inputs.scala }}.tar From 7d27b6a8cf132c87c327dfb9c02e78fafd831453 Mon Sep 17 00:00:00 2001 From: Arjun Sharda <77706434+ArjunSharda@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:46:10 -0500 Subject: [PATCH 2/5] Update build_and_test.yml --- .github/workflows/build_and_test.yml | 94 ++++++++++++++-------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ef2235c37493..896405f3f9c5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -58,7 +58,7 @@ jobs: required: ${{ steps.set-outputs.outputs.required }} steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -180,7 +180,7 @@ jobs: SPARK_LOCAL_IP: localhost steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -195,7 +195,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 2G limit. - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -206,18 +206,18 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: ${{ matrix.java }}-${{ matrix.hadoop }}-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | ${{ matrix.java }}-${{ matrix.hadoop }}-coursier- - name: Install Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - name: Install Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 # We should install one Python that is higher then 3+ for SQL and Yarn because: # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils. # - Yarn has a Python specific test too, for example, YarnClusterSuite. @@ -240,13 +240,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" - name: Upload test results to report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/unit-tests.log" @@ -283,7 +283,7 @@ jobs: METASPACE_SIZE: 1g steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -298,7 +298,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 2G limit. - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -309,14 +309,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | pyspark-coursier- - name: Install Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - name: List Python packages (Python 3.9, PyPy3) @@ -335,20 +335,20 @@ jobs: ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" - name: Upload coverage to Codecov if: inputs.type == 'pyspark-coverage-scheduled' - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: files: ./python/coverage.xml flags: unittests name: PySpark - name: Upload test results to report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: test-results-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: unit-tests-log-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -368,7 +368,7 @@ jobs: SKIP_MIMA: true steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -383,7 +383,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 2G limit. - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -394,14 +394,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: sparkr-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | sparkr-coursier- - name: Install Java ${{ inputs.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ inputs.java }} - name: Run tests @@ -414,7 +414,7 @@ jobs: ./dev/run-tests --parallelism 1 --modules sparkr - name: Upload test results to report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: test-results-sparkr--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" @@ -434,7 +434,7 @@ jobs: image: dongjoon/apache-spark-github-action-image:20220207 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -447,7 +447,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty # Cache local repositories. Note that GitHub Actions cache has a 2G limit. - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -458,14 +458,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: docs-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | docs-coursier- - name: Cache Maven local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository key: docs-maven-${{ hashFiles('**/pom.xml') }} @@ -513,7 +513,7 @@ jobs: cd docs bundle install - name: Install Java 8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 8 - name: Scala linter @@ -548,7 +548,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -560,7 +560,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -571,14 +571,14 @@ jobs: restore-keys: | build- - name: Cache Maven local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | java${{ matrix.java }}-maven- - name: Install Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - name: Build with Maven @@ -597,7 +597,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -609,7 +609,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -620,14 +620,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: scala-213-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | scala-213-coursier- - name: Install Java 8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 8 - name: Build with SBT @@ -644,7 +644,7 @@ jobs: SPARK_LOCAL_IP: localhost steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -656,7 +656,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -667,25 +667,25 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: tpcds-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | tpcds-coursier- - name: Install Java 8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 8 - name: Cache TPC-DS generated data id: cache-tpcds-sf-1 - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ./tpcds-sf-1 key: tpcds-${{ hashFiles('.github/workflows/build_and_test.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }} - name: Checkout tpcds-kit repository if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: databricks/tpcds-kit ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069 @@ -721,13 +721,13 @@ jobs: spark.sql.join.forceApplyShuffledHashJoin=true - name: Upload test results to report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: test-results-tpcds--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: unit-tests-log-tpcds--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" @@ -746,7 +746,7 @@ jobs: SKIP_MIMA: true steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -759,7 +759,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache Scala, SBT and Maven - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | build/apache-maven-* @@ -770,14 +770,14 @@ jobs: restore-keys: | build- - name: Cache Coursier local repository - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/coursier key: docker-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | docker-integration-coursier- - name: Install Java 8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 8 - name: Run tests @@ -785,13 +785,13 @@ jobs: ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest - name: Upload test results to report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: test-results-docker-integration--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: unit-tests-log-docker-integration--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" From 630addbf07298c11d095877b3f1444cfe8b9da10 Mon Sep 17 00:00:00 2001 From: Arjun Sharda <77706434+ArjunSharda@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:47:07 -0500 Subject: [PATCH 3/5] Update stale.yml --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f27067384455..cdb19e3e2615 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -27,7 +27,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@c201d45ef4b0ccbd3bb0616f93bae13e73d0a080 # pin@v1.1.0 + - uses: actions/stale@v5 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: > From c92ff7a1b8b9a9cabf8d0850e9cfcda85c6ab981 Mon Sep 17 00:00:00 2001 From: Arjun Sharda <77706434+ArjunSharda@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:47:35 -0500 Subject: [PATCH 4/5] Update labeler.yml --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 88d17bf34d50..9aeac7b546a7 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -44,7 +44,7 @@ jobs: # # However, these are not in a published release and the current `main` branch # has some issues upon testing. - - uses: actions/labeler@5f867a63be70efff62b767459b009290364495eb # pin@2.2.0 + - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" sync-labels: true From 82249e7855264ae27b01b557b4432dee1afd0b13 Mon Sep 17 00:00:00 2001 From: Arjun Sharda <77706434+ArjunSharda@users.noreply.github.com> Date: Sat, 2 Jul 2022 10:48:26 -0500 Subject: [PATCH 5/5] Update publish_snapshot.yml --- .github/workflows/publish_snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index d375a4149ded..7f1eaeaec28c 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -37,18 +37,18 @@ jobs: - branch-3.1 steps: - name: Checkout Spark repository - uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # pin@master + uses: actions/checkout@v3 with: ref: ${{ matrix.branch }} - name: Cache Maven local repository - uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353 # pin@v2 + uses: actions/cache@v3 with: path: ~/.m2/repository key: snapshot-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | snapshot-maven- - name: Install Java 8 - uses: actions/setup-java@d202f5dbf7256730fb690ec59f6381650114feb2 # pin@v1 + uses: actions/setup-java@v3 with: java-version: 8 - name: Publish snapshot