From bf064f14ee6e4c9b87894e99a91583eb3e9fdd12 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Fri, 3 Apr 2020 13:32:28 -0400 Subject: [PATCH 01/10] add autolabeler rules --- .github/autolabeler.yml | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/autolabeler.yml diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml new file mode 100644 index 000000000000..b932346ef23d --- /dev/null +++ b/.github/autolabeler.yml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Bot page: https://github.com/apps/probot-autolabeler +# The matching patterns follow the .gitignore spec. +# See: https://git-scm.com/docs/gitignore#_pattern_format + +infra: + - ".github/" + - "appveyor.yml" + - "/tools/" +build: + - "/dev/" + - "/build/" + - "/project/" +release: + - "/dev/create-release/" +docs: + - "docs/" + - "examples/" + - "/README.md" + - "/CONTRIBUTING.md" +core: + - "/core/" +sql: + - "/sql/" +ml: + - "/mllib/" + - "/mllib-local/" +streaming: + - "/streaing/" +python: + - "/python/" +java: + - "/common/" + - "java/" +R: + - "/R/" From 0d19a74bd592cd452eea5b2e5067f0dacce2dcc0 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Fri, 3 Apr 2020 13:43:36 -0400 Subject: [PATCH 02/10] tweak rules --- .github/autolabeler.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index b932346ef23d..13cee3a5553b 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -37,14 +37,15 @@ docs: core: - "/core/" sql: - - "/sql/" + - "sql/" ml: - - "/mllib/" - - "/mllib-local/" + - "ml/" + - "mllib/" + - "mllib-local/" streaming: - - "/streaing/" + - "streaming/" python: - - "/python/" + - "python/" java: - "/common/" - "java/" From d6ecb1acb13f7e451c9dd934df60534cdae29269 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Fri, 3 Apr 2020 14:00:46 -0400 Subject: [PATCH 03/10] tweak r --- .github/autolabeler.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 13cee3a5553b..75ddb094d58c 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -50,4 +50,5 @@ java: - "/common/" - "java/" R: - - "/R/" + - "r/" + - "R/" From 20b220c05b077b1c3905cd93f21b2a873d2f0324 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 10 Apr 2020 19:42:29 +0900 Subject: [PATCH 04/10] Refine the list and add some more fine-grained items --- .github/autolabeler.yml | 97 +++++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 75ddb094d58c..b3aef4c182d3 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -18,37 +18,98 @@ # Bot page: https://github.com/apps/probot-autolabeler # The matching patterns follow the .gitignore spec. # See: https://git-scm.com/docs/gitignore#_pattern_format - -infra: +# Also, note that the plugin uses 'ignore' package. See also +# https://github.com/kaelzhang/node-ignore +INFRA: - ".github/" - "appveyor.yml" - "/tools/" -build: + - "/dev/create-release/" + - ".asf.yaml" + - ".gitattributes" + - ".gitignore" + - "scalastyle-config.xml" +BUILD: - "/dev/" - "/build/" - "/project/" -release: - - "/dev/create-release/" -docs: + - "/assembly/" + - "*pom.xml" + - "/bin/docker-image-tool.sh" + - "/bin/find-spark-home*" +DOCS: - "docs/" - - "examples/" - "/README.md" - "/CONTRIBUTING.md" -core: +EXAMPLES: + - "examples/" + - "/bin/run-example*" +CORE: - "/core/" -sql: + - "/common/kvstore" + - "/common/network-common" + - "/common/network-shuffle" + - "/python/pyspark/*.py" + - "/python/pyspark/tests/*.py" + - "/sbin/*master*.sh" + - "/sbin/*slave*.sh" + - "/sbin/spark-config.sh" + - "/sbin/*daemon*.sh" + - "/sbin/*history*.sh" + - "/sbin/*mesos*.sh" +SPARK SUBMIT: + - "/bin/spark-submit*" + - "/bin/sparkR*" +SPARK SHELL: + - "/repl" +SQL: - "sql/" -ml: + - "/common/unsafe" + - "/python/pyspark/sql" + - "!/python/pyspark/sql/avro" + - "!/python/pyspark/sql/streaming.py" + - "!/python/pyspark/sql/tests/test_streaming.py" + - "/bin/spark-sql*" + - "/bin/beeline*" + - "/sbin/*thriftserver*.sh" +AVRO: + - /external/avro" + - "/python/pyspark/sql/avro" +DSTREAM: + - "/streaming/" + - "/data/streaming" + - "/external/flume/" + - "/external/flume-assembly/" + - "/external/flume-sink/" + - "/external/kinesis-asl/" + - "/external/kinesis-asl-assembly/" + - "/external/kafka-0-10/" + - "/external/kafka-0-10-assembly/" + - "/python/pyspark/streaming" +GRAPHX: + - "/graphx/" + - "/data/graphx" +ML: - "ml/" - - "mllib/" - - "mllib-local/" -streaming: - - "streaming/" -python: + - "/python/pyspark/ml" +MLLIB: + - "spark/mllib/" + - "/mllib-local" + - "/python/pyspark/mllib" +STRUCTURED STREAMING: + - "sql/**/streaming/" + - "/external/kafka-0-10-sql/" + - "/python/pyspark/sql/streaming.py" + - "/python/pyspark/sql/tests/test_streaming.py" +PYTHON: + - "/bin/pyspark*" - "python/" -java: - - "/common/" - - "java/" R: - "r/" - "R/" +YARN: + - "/resource-managers/yarn/" +MESOS: + - "/resource-managers/mesos/" +KUBERNETES: + - "/resource-managers/kubernetes/" From 84d0c327c4222a10284ed70a1e056c3766799244 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 10 Apr 2020 19:46:37 +0900 Subject: [PATCH 05/10] Move Scala style configuration to BUILD tag --- .github/autolabeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index b3aef4c182d3..a62c666b0bda 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -28,7 +28,6 @@ INFRA: - ".asf.yaml" - ".gitattributes" - ".gitignore" - - "scalastyle-config.xml" BUILD: - "/dev/" - "/build/" @@ -37,6 +36,7 @@ BUILD: - "*pom.xml" - "/bin/docker-image-tool.sh" - "/bin/find-spark-home*" + - "scalastyle-config.xml" DOCS: - "docs/" - "/README.md" From aadea310f856f00dbee961a929acc0a98d2468ff Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 10 Apr 2020 19:53:00 +0900 Subject: [PATCH 06/10] Fix another mistake --- .github/autolabeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index a62c666b0bda..687f03d5e741 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -59,7 +59,6 @@ CORE: - "/sbin/*mesos*.sh" SPARK SUBMIT: - "/bin/spark-submit*" - - "/bin/sparkR*" SPARK SHELL: - "/repl" SQL: @@ -107,6 +106,7 @@ PYTHON: R: - "r/" - "R/" + - "/bin/sparkR*" YARN: - "/resource-managers/yarn/" MESOS: From 8228cf412faea0c3f53abe9aaa3e7c6cf0e22c89 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 10 Apr 2020 20:01:24 +0900 Subject: [PATCH 07/10] Address my own comments --- .github/autolabeler.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 687f03d5e741..1ddd11e18d3a 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -72,18 +72,14 @@ SQL: - "/bin/beeline*" - "/sbin/*thriftserver*.sh" AVRO: - - /external/avro" + - "/external/avro" - "/python/pyspark/sql/avro" DSTREAM: - "/streaming/" - "/data/streaming" - - "/external/flume/" - - "/external/flume-assembly/" - - "/external/flume-sink/" - - "/external/kinesis-asl/" - - "/external/kinesis-asl-assembly/" - - "/external/kafka-0-10/" - - "/external/kafka-0-10-assembly/" + - "/external/flume*" + - "/external/kinesis*" + - "/external/kafka*" - "/python/pyspark/streaming" GRAPHX: - "/graphx/" From 7db3491ef5e8f4997a1420686eb6c28c41b208af Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Sat, 11 Apr 2020 12:21:37 +0900 Subject: [PATCH 08/10] Add some more compoenents, address comments --- .github/autolabeler.yml | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 1ddd11e18d3a..2fce6b48dca7 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -28,8 +28,14 @@ INFRA: - ".asf.yaml" - ".gitattributes" - ".gitignore" + - "/dev/github_jira_sync.py" + - "/dev/merge_spark_pr.py" + - "/dev/run-tests-jenkins*" BUILD: - "/dev/" + - "!/dev/github_jira_sync.py" + - "!/dev/merge_spark_pr.py" + - "!/dev/run-tests-jenkins*" - "/build/" - "/project/" - "/assembly/" @@ -46,9 +52,9 @@ EXAMPLES: - "/bin/run-example*" CORE: - "/core/" - - "/common/kvstore" - - "/common/network-common" - - "/common/network-shuffle" + - "/common/kvstore/" + - "/common/network-common/" + - "/common/network-shuffle/" - "/python/pyspark/*.py" - "/python/pyspark/tests/*.py" - "/sbin/*master*.sh" @@ -60,37 +66,35 @@ CORE: SPARK SUBMIT: - "/bin/spark-submit*" SPARK SHELL: - - "/repl" + - "/repl/" SQL: - "sql/" - - "/common/unsafe" - - "/python/pyspark/sql" - - "!/python/pyspark/sql/avro" + - "/common/unsafe/" + - "!/python/pyspark/sql/avro/" - "!/python/pyspark/sql/streaming.py" - "!/python/pyspark/sql/tests/test_streaming.py" - "/bin/spark-sql*" - "/bin/beeline*" - "/sbin/*thriftserver*.sh" AVRO: - - "/external/avro" - - "/python/pyspark/sql/avro" + - "/external/avro/" + - "/python/pyspark/sql/avro/" DSTREAM: - "/streaming/" - - "/data/streaming" + - "/data/streaming/" - "/external/flume*" - "/external/kinesis*" - "/external/kafka*" - - "/python/pyspark/streaming" + - "/python/pyspark/streaming/" GRAPHX: - "/graphx/" - - "/data/graphx" + - "/data/graphx/" ML: - "ml/" - - "/python/pyspark/ml" MLLIB: - "spark/mllib/" - - "/mllib-local" - - "/python/pyspark/mllib" + - "/mllib-local/" + - "/python/pyspark/mllib/" STRUCTURED STREAMING: - "sql/**/streaming/" - "/external/kafka-0-10-sql/" @@ -109,3 +113,8 @@ MESOS: - "/resource-managers/mesos/" KUBERNETES: - "/resource-managers/kubernetes/" +WINDOWS: + - "*.cmd" + - "/R/pkg/tests/fulltests/test_Windows.R" +WEB UI: + - "ui/" From 30a09c5eedaa2923bc221014c305922228fffb04 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Sat, 11 Apr 2020 12:42:13 +0900 Subject: [PATCH 09/10] Add more fine grained R files --- .github/autolabeler.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 2fce6b48dca7..2ba0c8a66d58 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -91,6 +91,7 @@ GRAPHX: - "/data/graphx/" ML: - "ml/" + - "*mllib_*.R" MLLIB: - "spark/mllib/" - "/mllib-local/" @@ -100,6 +101,7 @@ STRUCTURED STREAMING: - "/external/kafka-0-10-sql/" - "/python/pyspark/sql/streaming.py" - "/python/pyspark/sql/tests/test_streaming.py" + - "*streaming.R" PYTHON: - "/bin/pyspark*" - "python/" From 2ddde694c13f006d8bbb66bccd015497c53283d2 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Sat, 11 Apr 2020 12:48:56 +0900 Subject: [PATCH 10/10] Just add SQL for R components too --- .github/autolabeler.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml index 2ba0c8a66d58..e842090b1a29 100644 --- a/.github/autolabeler.yml +++ b/.github/autolabeler.yml @@ -76,6 +76,15 @@ SQL: - "/bin/spark-sql*" - "/bin/beeline*" - "/sbin/*thriftserver*.sh" + - "*SQL*.R" + - "DataFrame.R" + - "WindowSpec.R" + - "catalog.R" + - "column.R" + - "functions.R" + - "group.R" + - "schema.R" + - "types.R" AVRO: - "/external/avro/" - "/python/pyspark/sql/avro/"