From 257f78a2c50da33a62ae9deb7e19f6ce9a82e7db Mon Sep 17 00:00:00 2001 From: Huaxin Gao Date: Thu, 2 Apr 2020 00:05:47 -0700 Subject: [PATCH] [SPARK-31326][SQL][DOCS] Create Function docs structure for SQL Reference --- docs/_data/menu-sql.yaml | 21 ++++++++++++++++++++ docs/sql-ref-functions-builtin-aggregate.md | 10 +++++----- docs/sql-ref-functions-builtin-array.md | 22 +++++++++++++++++++++ docs/sql-ref-functions-builtin-date-time.md | 22 +++++++++++++++++++++ docs/sql-ref-functions-builtin.md | 17 ++++++++-------- docs/sql-ref-functions-udf-hive.md | 22 +++++++++++++++++++++ docs/sql-ref-functions-udf.md | 17 ++++++++-------- docs/sql-ref-functions.md | 13 +++++------- 8 files changed, 115 insertions(+), 29 deletions(-) create mode 100644 docs/sql-ref-functions-builtin-array.md create mode 100644 docs/sql-ref-functions-builtin-date-time.md create mode 100644 docs/sql-ref-functions-udf-hive.md diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml index 6534c50be2cc..500895aab4a6 100644 --- a/docs/_data/menu-sql.yaml +++ b/docs/_data/menu-sql.yaml @@ -225,5 +225,26 @@ url: sql-ref-syntax-aux-resource-mgmt-list-file.html - text: LIST JAR url: sql-ref-syntax-aux-resource-mgmt-list-jar.html + - text: Functions + url: sql-ref-functions.html + subitems: + - text: Build-in Functions + url: sql-ref-functions-builtin.html + subitems: + - text: Build-in Aggregate Functions + url: sql-ref-functions-builtin-aggregate.html + - text: Build-in Array Functions + url: sql-ref-functions-builtin-array.html + - text: Build-in Date Time Functions + url: sql-ref-functions-builtin-date-time.html + - text: UDFs (User-Defined Functions) + url: sql-ref-functions-udf.html + subitems: + - text: Scalar UDFs (User-Defined Functions) + url: sql-ref-functions-udf-scalar.html + - text: UDAFs (User-Defined Aggregate Functions) + url: sql-ref-functions-udf-aggregate.html + - text: Integration with Hive UDFs/UDAFs/UDTFs + url: sql-ref-functions-udf-hive.html - text: Datetime Pattern url: sql-ref-datetime-pattern.html diff --git a/docs/sql-ref-functions-builtin-aggregate.md b/docs/sql-ref-functions-builtin-aggregate.md index 3fcd78245ca0..d59543647e02 100644 --- a/docs/sql-ref-functions-builtin-aggregate.md +++ b/docs/sql-ref-functions-builtin-aggregate.md @@ -1,7 +1,7 @@ --- layout: global -title: Builtin Aggregate Functions -displayTitle: Builtin Aggregate Functions +title: Built-in Aggregate Functions +displayTitle: Built-in Aggregate Functions license: | Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -9,9 +9,9 @@ license: | The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ license: | limitations under the License. --- -**This page is under construction** +Aggregate functions \ No newline at end of file diff --git a/docs/sql-ref-functions-builtin-array.md b/docs/sql-ref-functions-builtin-array.md new file mode 100644 index 000000000000..599d10c23b61 --- /dev/null +++ b/docs/sql-ref-functions-builtin-array.md @@ -0,0 +1,22 @@ +--- +layout: global +title: Built-in Array Functions +displayTitle: Built-in Array Functions +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Array Functions diff --git a/docs/sql-ref-functions-builtin-date-time.md b/docs/sql-ref-functions-builtin-date-time.md new file mode 100644 index 000000000000..10421324a344 --- /dev/null +++ b/docs/sql-ref-functions-builtin-date-time.md @@ -0,0 +1,22 @@ +--- +layout: global +title: Built-in Date and Time Functions +displayTitle: Built-in Date and Time Functions +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Date-Time Functions diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 6c57b0d6fdf6..48e5c0e6e13e 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -1,7 +1,7 @@ --- layout: global -title: Reference -displayTitle: Reference +title: Built-in Functions +displayTitle: Built-in Functions license: | Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -9,9 +9,9 @@ license: | The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,8 @@ license: | limitations under the License. --- -Spark SQL is Apache Spark's module for working with structured data. -This guide is a reference for Structured Query Language (SQL) for Apache -Spark. This document describes the SQL constructs supported by Spark in detail -along with usage examples when applicable. +Spark SQL defines built-in functions to use, a complete list of which can be found [here](api/sql/). Among them, Spark SQL has several special categories of built-in functions: [Aggregate Functions](sql-ref-functions-builtin-aggregate.html) to operate on a group of rows, [Array Functions](sql-ref-functions-builtin-array.html) to operate on Array columns, and [Date and Time Functions](sql-ref-functions-builtin-date-time.html) to operate on Date and Time. + +* [Aggregate Functions](sql-ref-functions-builtin-aggregate.html) +* [Array Functions](sql-ref-functions-builtin-array.html) +* [Date and Time Functions](sql-ref-functions-builtin-date-time.html) diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md new file mode 100644 index 000000000000..8698be707ea4 --- /dev/null +++ b/docs/sql-ref-functions-udf-hive.md @@ -0,0 +1,22 @@ +--- +layout: global +title: Integration with Hive UDFs/UDAFs/UDTFs +displayTitle: Integration with Hive UDFs/UDAFs/UDTFs +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +Integration with Hive UDFs/UDAFs/UDTFs \ No newline at end of file diff --git a/docs/sql-ref-functions-udf.md b/docs/sql-ref-functions-udf.md index 6c57b0d6fdf6..91c04f16049e 100644 --- a/docs/sql-ref-functions-udf.md +++ b/docs/sql-ref-functions-udf.md @@ -1,7 +1,7 @@ --- layout: global -title: Reference -displayTitle: Reference +title: UDFs (User-Defined Functions) +displayTitle: UDFs (User-Defined Functions) license: | Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -9,9 +9,9 @@ license: | The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,8 @@ license: | limitations under the License. --- -Spark SQL is Apache Spark's module for working with structured data. -This guide is a reference for Structured Query Language (SQL) for Apache -Spark. This document describes the SQL constructs supported by Spark in detail -along with usage examples when applicable. +User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs. + +* [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html) +* [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html) +* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html) diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md index 6c57b0d6fdf6..dabf90bc5285 100644 --- a/docs/sql-ref-functions.md +++ b/docs/sql-ref-functions.md @@ -1,7 +1,7 @@ --- layout: global -title: Reference -displayTitle: Reference +title: Functions +displayTitle: Functions license: | Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -9,9 +9,9 @@ license: | The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,7 +19,4 @@ license: | limitations under the License. --- -Spark SQL is Apache Spark's module for working with structured data. -This guide is a reference for Structured Query Language (SQL) for Apache -Spark. This document describes the SQL constructs supported by Spark in detail -along with usage examples when applicable. +Spark SQL defines commonly used functions as built-in functions. It also provides User-Defined Functions (UDFs) to allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.