diff --git a/common/tags/src/test/java/org/apache/spark/tags/DockerTest.java b/common/tags/src/test/java/org/apache/spark/tags/DockerTest.java deleted file mode 100644 index 0fecf3b8f979..000000000000 --- a/common/tags/src/test/java/org/apache/spark/tags/DockerTest.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.tags; - -import java.lang.annotation.*; -import org.scalatest.TagAnnotation; - -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface DockerTest { } diff --git a/common/tags/src/test/java/org/apache/spark/tags/ExtendedHiveTest.java b/common/tags/src/test/java/org/apache/spark/tags/ExtendedHiveTest.java deleted file mode 100644 index 83279e5e93c0..000000000000 --- a/common/tags/src/test/java/org/apache/spark/tags/ExtendedHiveTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.tags; - -import java.lang.annotation.*; - -import org.scalatest.TagAnnotation; - -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface ExtendedHiveTest { } diff --git a/common/tags/src/test/java/org/apache/spark/tags/ExtendedYarnTest.java b/common/tags/src/test/java/org/apache/spark/tags/ExtendedYarnTest.java deleted file mode 100644 index 108300168e17..000000000000 --- a/common/tags/src/test/java/org/apache/spark/tags/ExtendedYarnTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.tags; - -import java.lang.annotation.*; - -import org.scalatest.TagAnnotation; - -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface ExtendedYarnTest { } diff --git a/core/pom.xml b/core/pom.xml index 9258a856028a..0417c3dbf853 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -359,17 +359,6 @@ test - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - org.apache.commons commons-crypto diff --git a/dev/run-tests.py b/dev/run-tests.py index 7e6f7ff06035..c1b5d4fbfd35 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -124,14 +124,6 @@ def determine_modules_to_test(changed_modules): {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True) -def determine_tags_to_exclude(changed_modules): - tags = [] - for m in modules.all_modules: - if m not in changed_modules: - tags += m.test_tags - return tags - - # ------------------------------------------------------------------------------------------------- # Functions for working with subprocesses and shell tools # ------------------------------------------------------------------------------------------------- @@ -423,7 +415,7 @@ def run_scala_tests_sbt(test_modules, test_profiles): exec_sbt(profiles_and_goals) -def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags): +def run_scala_tests(build_tool, hadoop_version, test_modules): """Function to properly execute all tests passed in as a set from the `determine_test_suites` function""" set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS") @@ -433,9 +425,6 @@ def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags): test_profiles = get_hadoop_profiles(hadoop_version) + \ list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules))) - if excluded_tags: - test_profiles += ['-Dtest.exclude.tags=' + ",".join(excluded_tags)] - if build_tool == "maven": run_scala_tests_maven(test_profiles) else: @@ -544,10 +533,8 @@ def main(): target_branch = os.environ["ghprbTargetBranch"] changed_files = identify_changed_files_from_git_commits("HEAD", target_branch=target_branch) changed_modules = determine_modules_for_files(changed_files) - excluded_tags = determine_tags_to_exclude(changed_modules) if not changed_modules: changed_modules = [modules.root] - excluded_tags = [] print("[info] Found the following changed modules:", ", ".join(x.name for x in changed_modules)) @@ -601,7 +588,7 @@ def main(): build_spark_assembly_sbt(hadoop_version) # run the test suites - run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags) + run_scala_tests(build_tool, hadoop_version, test_modules) modules_with_python_tests = [m for m in test_modules if m.python_test_goals] if modules_with_python_tests: diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 7164180a6a7b..812633ffec4d 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -33,7 +33,7 @@ class Module(object): def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={}, sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(), - test_tags=(), should_run_r_tests=False, should_run_build_tests=False): + should_run_r_tests=False, should_run_build_tests=False): """ Define a new module. @@ -52,8 +52,6 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags= :param blacklisted_python_implementations: A set of Python implementations that are not supported by this module's Python components. The values in this set should match strings returned by Python's `platform.python_implementation()`. - :param test_tags A set of tags that will be excluded when running unit tests if the module - is not explicitly changed. :param should_run_r_tests: If true, changes in this module will trigger all R tests. :param should_run_build_tests: If true, changes in this module will trigger build tests. """ @@ -65,7 +63,6 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags= self.environ = environ self.python_test_goals = python_test_goals self.blacklisted_python_implementations = blacklisted_python_implementations - self.test_tags = test_tags self.should_run_r_tests = should_run_r_tests self.should_run_build_tests = should_run_build_tests @@ -137,9 +134,6 @@ def __hash__(self): sbt_test_goals=[ "hive/test", ], - test_tags=[ - "org.apache.spark.tags.ExtendedHiveTest" - ] ) @@ -523,9 +517,6 @@ def __hash__(self): "yarn/test", "network-yarn/test", ], - test_tags=[ - "org.apache.spark.tags.ExtendedYarnTest" - ] ) mesos = Module( diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 431339d41219..e6be89666582 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -100,12 +100,6 @@ test-jar test - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - mysql mysql-connector-java diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala index f5930bc281e8..23dd234c8c74 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala @@ -25,10 +25,8 @@ import org.scalatest.Ignore import org.apache.spark.sql.Row import org.apache.spark.sql.types.{BooleanType, ByteType, ShortType, StructType} -import org.apache.spark.tags.DockerTest -@DockerTest @Ignore // AMPLab Jenkins needs to be updated before shared memory works on docker class DB2IntegrationSuite extends DockerJDBCIntegrationSuite { override val db = new DatabaseOnDocker { diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala index a70ed98b52d5..1f857d6516b7 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala @@ -21,9 +21,6 @@ import java.math.BigDecimal import java.sql.{Connection, Date, Timestamp} import java.util.Properties -import org.apache.spark.tags.DockerTest - -@DockerTest class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite { override val db = new DatabaseOnDocker { override val imageName = "mysql:5.7.9" diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 8512496e5fe5..57eef30486b9 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.execution.{RowDataSourceScanExec, WholeStageCodegenE import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.sql.types._ -import org.apache.spark.tags.DockerTest /** * This patch was tested using the Oracle docker. Created this integration suite for the same. @@ -48,7 +47,6 @@ import org.apache.spark.tags.DockerTest * All tests in this suite are ignored because of the dependency with the oracle jar from maven * repository. */ -@DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLContext { import testImplicits._ diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala index be32cb89f488..add5e8870feb 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala @@ -23,9 +23,7 @@ import java.util.Properties import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.types.{ArrayType, DecimalType, FloatType, ShortType} -import org.apache.spark.tags.DockerTest -@DockerTest class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite { override val db = new DatabaseOnDocker { override val imageName = "postgres:9.4.5" diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index f810aa80e878..77043488377e 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -94,17 +94,6 @@ spark-tags_${scala.binary.version} - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - target/scala-${scala.binary.version}/classes diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 498e88f665eb..9dd8999fd196 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -69,18 +69,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - target/scala-${scala.binary.version}/classes diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 16bbc6db641c..23bf172ff31f 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -89,18 +89,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 3b124b2a69d5..d11c14b1e8ba 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -74,18 +74,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index 6d1c4789f382..d20af31e522a 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -89,18 +89,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - target/scala-${scala.binary.version}/classes diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 491589396559..efc013a70e62 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -83,18 +83,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - target/scala-${scala.binary.version}/classes diff --git a/graphx/pom.xml b/graphx/pom.xml index fbe77fcb958d..c1036679a797 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -78,18 +78,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - target/scala-${scala.binary.version}/classes diff --git a/launcher/pom.xml b/launcher/pom.xml index 912eb6b6d2a0..b0e3fd9210ce 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -67,17 +67,6 @@ spark-tags_${scala.binary.version} - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - org.apache.hadoop diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 53286fe93478..881c5c584235 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -56,18 +56,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - diff --git a/mllib/pom.xml b/mllib/pom.xml index f07d7f24fd31..a00579a3b099 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -125,18 +125,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - diff --git a/pom.xml b/pom.xml index 666d5d7169a1..5ee2716dd39a 100644 --- a/pom.xml +++ b/pom.xml @@ -188,7 +188,6 @@ 0.8.0 ${java.home} - org.spark_project @@ -2144,7 +2143,6 @@ src false - ${test.exclude.tags} @@ -2191,7 +2189,6 @@ __not_used__ - ${test.exclude.tags} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7469f11df029..5cff6c0204b3 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -807,15 +807,6 @@ object TestSettings { javaOptions in Test ++= "-Xmx3g -Xss4m" .split(" ").toSeq, javaOptions += "-Xmx3g", - // Exclude tags defined in a system property - testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, - sys.props.get("test.exclude.tags").map { tags => - tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq - }.getOrElse(Nil): _*), - testOptions in Test += Tests.Argument(TestFrameworks.JUnit, - sys.props.get("test.exclude.tags").map { tags => - Seq("--exclude-categories=" + tags) - }.getOrElse(Nil): _*), // Show full stack trace and duration in test cases. testOptions in Test += Tests.Argument("-oDF"), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), diff --git a/repl/pom.xml b/repl/pom.xml index 6f4a863c48bc..41174c3421fe 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -88,18 +88,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - org.apache.xbean xbean-asm5-shaded diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index 061f653b97b7..ab158118e459 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -40,7 +40,6 @@ import org.apache.spark.launcher._ import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationStart, SparkListenerExecutorAdded} import org.apache.spark.scheduler.cluster.ExecutorInfo -import org.apache.spark.tags.ExtendedYarnTest import org.apache.spark.util.Utils /** @@ -48,7 +47,6 @@ import org.apache.spark.util.Utils * applications, and require the Spark assembly to be built before they can be successfully * run. */ -@ExtendedYarnTest class YarnClusterSuite extends BaseYarnClusterSuite { override def newYarnConfig(): YarnConfiguration = new YarnConfiguration() diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala index 01db796096f2..2b57009a3f3b 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala @@ -31,12 +31,10 @@ import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.network.shuffle.ShuffleTestAccessor import org.apache.spark.network.yarn.{YarnShuffleService, YarnTestAccessor} -import org.apache.spark.tags.ExtendedYarnTest /** * Integration test for the external shuffle service with a yarn mini-cluster */ -@ExtendedYarnTest class YarnShuffleIntegrationSuite extends BaseYarnClusterSuite { override def newYarnConfig(): YarnConfiguration = { @@ -88,7 +86,6 @@ class YarnShuffleIntegrationSuite extends BaseYarnClusterSuite { /** * Integration test for the external shuffle service with auth on. */ -@ExtendedYarnTest class YarnShuffleAuthSuite extends YarnShuffleIntegrationSuite { override def newYarnConfig(): YarnConfiguration = { diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 7d23637e2834..52d2dd3b4776 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -60,18 +60,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - org.apache.spark spark-unsafe_${scala.binary.version} diff --git a/sql/core/pom.xml b/sql/core/pom.xml index ef41837f89d6..6d74819c9ea9 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -74,18 +74,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - org.apache.orc orc-core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 9f247f9224c7..f045499c5d15 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -95,18 +95,6 @@ org.apache.spark spark-tags_${scala.binary.version} - - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - net.sf.jpam jpam diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index c55ba32fa458..e8a5ad6b0fef 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -71,12 +71,6 @@ ${project.version} test - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - - org.apache.spark - spark-tags_${scala.binary.version} - test-jar - test - - com.google.guava