Operator 0.1.0

.editorconfig

@@ -0,0 +1,2 @@
+[*]
+insert_final_newline = true

.gitattributes

@@ -0,0 +1,5 @@
+#
+# https://help.github.com/articles/dealing-with-line-endings/
+#
+# Linux start script should use lf
+/gradlew        text eol=lf

.github/.licenserc.yaml

@@ -9,12 +9,19 @@ header:
   paths-ignore:
     - '**/*.md'
     - '**/.gitignore'
+    - '**/.gitattributes'
     - '.github/**'
     - 'dev/**'
     - 'LICENSE'
     - 'NOTICE'
     - '.asf.yaml'
     - '**/*.gradle'
     - gradlew
+    - '**/.helmignore'
+    - '**/EcsLayout.json'
+    - '.editorconfig'
+    - 'gradle/**'
+    - '**/sparkapplications.org.apache.spark-v1.yml'
+
 
   comment: on-failure

.github/workflows/build_and_test.yml

@@ -26,4 +26,20 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           config: .github/.licenserc.yaml
-
+  test_ci:
+    name: "Test CI"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 11, 17, 21 ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Set up JDK ${{ matrix.java-version }}
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'adopt'
+      - name: Build with Gradle
+        run: |
+          set -o pipefail; ./gradlew clean build; set +o pipefail

.gitignore

@@ -16,3 +16,30 @@ build
 dependencies.lock
 **/dependencies.lock
 gradle/wrapper/gradle-wrapper.jar
+
+# Compiled source #
+###################
+*.class
+*.dll
+*.exe
+*.o
+*.so
+*.pyc
+
+# Packages #
+############
+*.7z
+*.dmg
+*.gz
+*.iso
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+
+# Other build and generated files #
+###################################
+build-tools/helm/spark-kubernetes-operator/crds/

Dockerfile

@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+FROM gradle:8.7-jdk17-alpine AS builder
+ARG BASE_VERSION
+WORKDIR /app
+COPY . .
+RUN ./gradlew clean build -x test
+
+FROM eclipse-temurin:17-jre-jammy
+ARG BASE_VERSION
+
+ENV SPARK_OPERATOR_HOME=/opt/spark-operator
+ENV SPARK_OPERATOR_WORK_DIR=/opt/spark-operator/operator
+ENV BASE_VERSION=$BASE_VERSION
+ENV OPERATOR_JAR=spark-kubernetes-operator-$BASE_VERSION-all.jar
+
+WORKDIR $SPARK_OPERATOR_WORK_DIR
+
+RUN groupadd --system --gid=9999 spark && \
+    useradd --system --home-dir $SPARK_OPERATOR_HOME --uid=9999 --gid=spark spark
+
+COPY --from=builder /app/spark-operator/build/libs/$OPERATOR_JAR .
+COPY docker-entrypoint.sh .
+
+RUN chown -R spark:spark $SPARK_OPERATOR_HOME && \
+    chown spark:spark $OPERATOR_JAR && \
+    chown spark:spark docker-entrypoint.sh
+
+USER spark
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["help"]

README.md

@@ -1 +1,44 @@
-# spark-kubernetes-operator
+# Spark-Kubernetes-Operator
+
+Welcome to the **Spark-Kubernetes-Operator**, a Kubernetes operator designed to simplify and
+automate the management of Spark applications in Kubernetes environments.
+
+## Project Status
+
+As of Apr 1, 2024, Spark-Kubernetes-Operator is under Active Development.
+
+- We are actively working on new features and improvements. We welcome contributions and
+  feedback to make the operator even better. Check out the **Issues** section to see what's
+  currently in progress or suggest new features.
+- Current API Version: `v1alpha1`
+
+## Key Features
+
+- Deploy and monitor SparkApplications throughout its lifecycle
+- Start / stop Spark Apps with simple yaml schema
+- Spark version agnostic
+- Full logging and metrics integration
+- Flexible deployments and native integration with Kubernetes tooling
+
+Please refer the [design](spark-operator-docs/architecture.md) section for architecture and 
+design.
+
+## Quickstart
+
+[Getting started doc](./spark-operator-docs/getting_started.md) gives an example to install 
+operator and run Spark Applications locally.
+
+In addition, [SparkApplication](./spark-operator-docs/spark_application.md) section 
+describes how to write your own apps, [Operations](./spark-operator-docs/operations.md) section 
+describes how to install operator with custom config overriding.
+
+
+
+## Contributing
+
+You can learn more about how to contribute in the [Apache Spark website](https://spark.
+apache.org/contributing.html). 
+
+## License
+
+The code in this repository is licensed under the [Apache Software License 2](./LICENSE).

build-tools/helm/spark-kubernetes-operator/.helmignore

@@ -0,0 +1,18 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

build-tools/helm/spark-kubernetes-operator/Chart.yaml

@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+---
+apiVersion: v2
+name: spark-kubernetes-operator
+description: A Helm chart for the Apache Spark Kubernetes Operator
+type: application
+version: 0.1.0
+appVersion: 0.1.0

build-tools/helm/spark-kubernetes-operator/conf/log4j2.properties

@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+status=debug
+strict=true
+dest=out
+name=PropertiesConfig
+property.filename=/opt/spark-operator/logs/spark-operator
+filter.threshold.type=ThresholdFilter
+filter.threshold.level=debug
+# console
+appender.console.type=Console
+appender.console.name=STDOUT
+appender.console.layout.type=PatternLayout
+appender.console.layout.pattern=%d %p %X %C{1.} [%t] %m%n
+appender.console.filter.threshold.type=ThresholdFilter
+appender.console.filter.threshold.level=info
+# rolling JSON
+appender.rolling.type=RollingFile
+appender.rolling.name=RollingFile
+appender.rolling.append=true
+appender.rolling.fileName=${filename}.log
+appender.rolling.filePattern=${filename}-%i.log.gz
+appender.rolling.layout.type=JsonTemplateLayout
+appender.rolling.layout.eventTemplateUri=classpath:EcsLayout.json
+appender.rolling.policies.type=Policies
+appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=100MB
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.max=20
+appender.rolling.immediateFlush=true
+# chatty loggers
+rootLogger.level=all
+logger.netty.name=io.netty
+logger.netty.level=warn
+log4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
+rootLogger.appenderRef.stdout.ref=STDOUT
+rootLogger.appenderRef.rolling.ref=RollingFile

build-tools/helm/spark-kubernetes-operator/conf/spark-operator.properties

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Property Overrides. e.g.
+# kubernetes.operator.reconcile.interval=15s
+# Enable this for hot property loading
+# spark.operator.dynamic.config.enabled=false