From ae68d5b11c883bf998e5b66b09b085abfffa8432 Mon Sep 17 00:00:00 2001 From: gp510 Date: Wed, 7 Mar 2018 14:59:10 -0800 Subject: [PATCH 1/3] Updated naming throughout project per issue #117 --- .gitignore | 2 +- README.md | 54 +++++++++---------- build.sh | 16 +++--- ci/Jenkinsfile | 2 +- ci/kafka_orca_gen.py | 2 +- ci/run_bastion.sh | 8 +-- ci/run_kafka_connect.sh | 12 ++--- .../connect-distributed-quickstart.properties | 8 +-- config/connect-distributed.properties | 8 +-- dependency-reduced-pom.xml | 6 +-- pom.xml | 6 +-- ...ect-splunk.iml => splunk-kafka-connect.iml | 1 + .../kafka/connect/SplunkSinkConnector.java | 6 +-- .../splunk/kafka/connect/SplunkSinkTask.java | 4 +- src/main/resources/version.properties | 6 +-- 15 files changed, 71 insertions(+), 70 deletions(-) rename kafka-connect-splunk.iml => splunk-kafka-connect.iml (97%) diff --git a/.gitignore b/.gitignore index f24242e3..6904954b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ hs_err_pid* target/* .idea/* -kafka-connect-splunk/ +splunk-kafka-connect/ pom.xml.versionsBackup .classpath .project \ No newline at end of file diff --git a/README.md b/README.md index daa1b5a3..7666c492 100644 --- a/README.md +++ b/README.md @@ -17,25 +17,25 @@ A Kafka Connect Sink for Splunk features: ## Build -1. Clone the repo from https://github.com/splunk/kafka-connect-splunk +1. Clone the repo from https://github.com/splunk/splunk-kafka-connect 2. Verify that Java8 JRE or JDK is installed. 3. Run `bash build.sh`. The build script will download all dependencies and build the Splunk Kafka Connector. -Note: The resulting "kafka-connect-splunk-*.tar.gz" package is self-contained. Bundled within it are the Kafka Connect framework, all 3rd party libraries, and the Splunk Kafka Connector. +Note: The resulting "splunk-kafka-connect-*.tar.gz" package is self-contained. Bundled within it are the Kafka Connect framework, all 3rd party libraries, and the Splunk Kafka Connector. ## Quick Start 1. [Start](https://kafka.apache.org/quickstart) your Kafka Cluster and confirm it is running. 2. If this is a new install, create a test topic (eg: `perf`). Inject events into the topic. This can be done using [Kafka data-gen-app](https://github.com/dtregonning/kafka-data-gen) or the Kafka bundle [kafka-console-producer](https://kafka.apache.org/quickstart#quickstart_send). -3. Untar the package created from the build script: `tar xzvf kafka-connect-splunk-*.tar.gz` (Default target location is /tmp/kafka-connect-splunk-build/kafka-connect-splunk). -4. Navigate to kafka-connect-splunk directory `cd kafka-connect-splunk`. +3. Untar the package created from the build script: `tar xzvf splunk-kafka-connect-*.tar.gz` (Default target location is /tmp/splunk-kafka-connect-build/splunk-kafka-connect). +4. Navigate to splunk-kafka-connect directory `cd splunk-kafka-connect`. 5. Adjust values for `bootstrap.servers` and `plugin.path` inside `config/connect-distributed-quickstart.properties` to fit your environment. Default values should work for experimentation. 6. Run `./bin/connect-distributed.sh config/connect-distributed-quickstart.properties` to start Kafka Connect. 7. Run the following command to create connector tasks. Adjust `topics` to set the topic, and `splunk.hec.token` to set your HEC token. ``` curl localhost:8083/connectors -X POST -H "Content-Type: application/json" -d '{ - "name": "kafka-connect-splunk", + "name": "splunk-kafka-connect", "config": { "connector.class": "com.splunk.kafka.connect.SplunkSinkConnector", "tasks.max": "3", @@ -71,17 +71,17 @@ Note: The resulting "kafka-connect-splunk-*.tar.gz" package is self-contained. B # List active connectors curl http://localhost:8083/connectors - # Get kafka-connect-splunk connector info - curl http://localhost:8083/connectors/kafka-connect-splunk + # Get splunk-kafka-connect connector info + curl http://localhost:8083/connectors/splunk-kafka-connect - # Get kafka-connect-splunk connector config info - curl http://localhost:8083/connectors/kafka-connect-splunk/config + # Get splunk-kafka-connect connector config info + curl http://localhost:8083/connectors/splunk-kafka-connect/config - # Delete kafka-connect-splunk connector - curl http://localhost:8083/connectors/kafka-connect-splunk -X DELETE + # Delete splunk-kafka-connect connector + curl http://localhost:8083/connectors/splunk-kafka-connect -X DELETE - # Get kafka-connect-splunk connector task info - curl http://localhost:8083/connectors/kafka-connect-splunk/tasks + # Get splunk-kafka-connect connector task info + curl http://localhost:8083/connectors/splunk-kafka-connect/tasks ``` See the [the Confluent doucumentation](https://docs.confluent.io/current/connect/managing.html#common-rest-examples) for additional REST examples. @@ -98,11 +98,11 @@ Use the following connector deployment options: ### Connector in a dedicated Kafka Connect Cluster Running the Splunk Kafka Connector in a dedicated Kafka Connect Cluster is recommended. Isolating the Splunk connector from other Kafka connectors results in significant performance benefits in high throughput environments. -1. Untar the **kafka-connect-splunk-*.tar.gz** package and navigate to the **kafka-connect-splunk** directory. +1. Untar the **splunk-kafka-connect-*.tar.gz** package and navigate to the **splunk-kafka-connect** directory. ``` - tar xzvf kafka-connect-splunk-*.tar.gz - cd kafka-connect-splunk + tar xzvf splunk-kafka-connect-*.tar.gz + cd splunk-kafka-connect ``` 2. Update config/connect-distributed.properties to match your environment. @@ -118,26 +118,26 @@ Running the Splunk Kafka Connector in a dedicated Kafka Connect Cluster is recom > Note: The below topics should be created by Kafka Connect when deploying the Splunk Connector. If the Kafka Connect cluster **does not have permission** to create these topics, create these manually before starting Kafka Connect cluster. ``` - group.id=kafka-connect-splunk-hec-sink # consumer group id of Kafka Connect, which is used to form a Kafka Connect cluster + group.id=splunk-kafka-connect-hec-sink # consumer group id of Kafka Connect, which is used to form a Kafka Connect cluster - config.storage.topic=__kafka-connect-splunk-task-configs # kafka topic used to persistent connector task configurations + config.storage.topic=__splunk-kafka-connect-task-configs # kafka topic used to persistent connector task configurations config.storage.replication.factor=3 - offset.storage.topic=__kafka-connect-splunk-offsets # kafka topic used to persistent task checkpoints + offset.storage.topic=__splunk-kafka-connect-offsets # kafka topic used to persistent task checkpoints offset.storage.replication.factor=3 offset.storage.partitions=25 - status.storage.topic=__kafka-connect-splunk-statuses # kafka topic used to persistent task statuses + status.storage.topic=__splunk-kafka-connect-statuses # kafka topic used to persistent task statuses status.storage.replication.factor=3 status.storage.partitions=5 ``` -4. Deploy/Copy the **kafka-connect-splunk** directory to all target hosts (virtual machines, physical machines or containers). +4. Deploy/Copy the **splunk-kafka-connect** directory to all target hosts (virtual machines, physical machines or containers). 5. Start Kafka Connect on all target hosts using the below commands: ``` - cd kafka-connect-splunk - export KAFKA_HEAP_OPTS="-Xmx6G -Xms2G" && ./bin/connect-distributed.sh config/connect-distributed.properties >> kafka-connect-splunk.log 2>&1 + cd splunk-kafka-connect + export KAFKA_HEAP_OPTS="-Xmx6G -Xms2G" && ./bin/connect-distributed.sh config/connect-distributed.properties >> splunk-kafka-connect.log 2>&1 ``` > Note: The **KAFKA\_HEAP\_OPTS** environment variable controls how much memory Kafka Connect can use. Set the **KAFKA\_HEAP\_OPTS** with the recommended value stated in the example above. @@ -167,13 +167,13 @@ internal.value.converter.schemas.enable=false offset.flush.interval.ms=10000 #Recommended -group.id=kafka-connect-splunk-hec-sink -config.storage.topic=__kafka-connect-splunk-task-configs +group.id=splunk-kafka-connect-hec-sink +config.storage.topic=__splunk-kafka-connect-task-configs config.storage.replication.factor=3 -offset.storage.topic=__kafka-connect-splunk-offsets +offset.storage.topic=__splunk-kafka-connect-offsets offset.storage.replication.factor=3 offset.storage.partitions=25 -status.storage.topic=__kafka-connect-splunk-statuses +status.storage.topic=__splunk-kafka-connect-statuses status.storage.replication.factor=3 status.storage.partitions=5 diff --git a/build.sh b/build.sh index 0cd81a29..4b579995 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ # variables kafkaversion=0.11.0.2 -builddir=/tmp/kafka-connect-splunk-build/kafka-connect-splunk +builddir=/tmp/splunk-kafka-connect-build/splunk-kafka-connect githash=`git rev-parse --short HEAD 2>/dev/null | sed "s/\(.*\)/@\1/"` # get current git hash gitbranch=`git rev-parse --abbrev-ref HEAD` # get current git branch @@ -15,7 +15,7 @@ if [[ -z "$gitversion" ]]; then jarversion=${gitversion}-SNAPSHOT fi -packagename=kafka-connect-splunk-${gitversion}.tar.gz +packagename=splunk-kafka-connect-${gitversion}.tar.gz # record git info in version.properties file under resources folder resourcedir='src/main/resources' @@ -39,8 +39,8 @@ mvn versions:set -DnewVersion=${jarversion} mvn package > /dev/null # Copy over the pacakge -echo "Copy over kafka-connect-splunk jar ..." -cp target/kafka-connect-splunk-${jarversion}.jar ${builddir}/connectors +echo "Copy over splunk-kafka-connect jar ..." +cp target/splunk-kafka-connect-${jarversion}.jar ${builddir}/connectors cp config/* ${builddir}/config cp README.md ${builddir} cp LICENSE ${builddir} @@ -64,19 +64,19 @@ echo "Clean up ..." # Package up echo "Package ${packagename} ..." -cd .. && tar czf ${packagename} kafka-connect-splunk +cd .. && tar czf ${packagename} splunk-kafka-connect echo "Copy package ${packagename} to ${curdir} ..." cp ${packagename} ${curdir} -/bin/rm -rf kafka-connect-splunk ${packagename} +/bin/rm -rf splunk-kafka-connect ${packagename} echo "Done with build & packaging" echo cat << EOP -To run the kafka-connect-splunk, do the following steps: -1. untar the package: tar xzf kafka-connect-splunk.tar.gz +To run the splunk-kafka-connect, do the following steps: +1. untar the package: tar xzf splunk-kafka-connect.tar.gz 2. config config/connect-distributed.properties according to your env 3. run: bash bin/connect-distributed.sh config/connect-distributed.properties 4. Use Kafka Connect REST api to create data collection tasks diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 4582d35d..b3c8a0f1 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -8,7 +8,7 @@ def dockerReq = new DockerRequest(steps, env, [imageName: "repo.splunk.com/splunk/products/splact:1.0.9", userId: "10777", - repoName: "git@github.com:splunk/kafka-connect-splunk.git", + repoName: "git@github.com:splunk/splunk-kafka-connect.git", runner: "yarn", remotePath: "/build"]) diff --git a/ci/kafka_orca_gen.py b/ci/kafka_orca_gen.py index 2e9c00b4..5099ab0d 100644 --- a/ci/kafka_orca_gen.py +++ b/ci/kafka_orca_gen.py @@ -6,7 +6,7 @@ DATA_GEN_IMAGE = 'repo.splunk.com/kafka-data-gen:0.4' KAFKA_IMAGE = 'repo.splunk.com/kafka-cluster:0.12' -KAFKA_CONNECT_IMAGE = 'repo.splunk.com/kafka-connect-splunk:1.8' +KAFKA_CONNECT_IMAGE = 'repo.splunk.com/splunk-kafka-connect:1.8' KAFKA_BASTION_IMAGE = 'repo.splunk.com/kafka-bastion:1.8' diff --git a/ci/run_bastion.sh b/ci/run_bastion.sh index 3f07507b..76b2279d 100755 --- a/ci/run_bastion.sh +++ b/ci/run_bastion.sh @@ -1,15 +1,15 @@ #!/bin/bash curdir=`pwd` -git clone git@github.com:splunk/kafka-connect-splunk.git +git clone git@github.com:splunk/splunk-kafka-connect.git branch=${KAFKA_CONNECT_BRANCH:-develop} -cd kafka-connect-splunk && git checkout ${branch} +cd splunk-kafka-connect && git checkout ${branch} duration=${SLEEP:-600} sleep ${duration} -bash ${curdir}/kafka-connect-splunk/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & +bash ${curdir}/splunk-kafka-connect/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & -python ${curdir}/kafka-connect-splunk/ci/perf.py +python ${curdir}/splunk-kafka-connect/ci/perf.py tail -f /dev/null diff --git a/ci/run_kafka_connect.sh b/ci/run_kafka_connect.sh index 8f62593b..8472140d 100755 --- a/ci/run_kafka_connect.sh +++ b/ci/run_kafka_connect.sh @@ -1,17 +1,17 @@ #!/bin/bash -# Checkout, build and run kafka-connect-splunk in the fight +# Checkout, build and run splunk-kafka-connect in the fight curdir=`pwd` -git clone git@github.com:splunk/kafka-connect-splunk.git +git clone git@github.com:splunk/splunk-kafka-connect.git branch=${KAFKA_CONNECT_BRANCH:-develop} # build the package -cd kafka-connect-splunk && git checkout ${branch} && bash build.sh +cd splunk-kafka-connect && git checkout ${branch} && bash build.sh # untar the package -tar xzf kafka-connect-splunk*.tar.gz -cd kafka-connect-splunk +tar xzf splunk-kafka-connect*.tar.gz +cd splunk-kafka-connect sed -i"" "s@bootstrap.servers=.*@bootstrap.servers=$KAFKA_BOOTSTRAP_SERVERS@g" config/connect-distributed.properties @@ -24,7 +24,7 @@ duration=${SLEEP:-300} sleep ${duration} echo "Run fix hosts" -bash ${curdir}/kafka-connect-splunk/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & +bash ${curdir}/splunk-kafka-connect/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & echo "Run proc monitor" cd proc_monitor diff --git a/config/connect-distributed-quickstart.properties b/config/connect-distributed-quickstart.properties index d11e119e..d05ad598 100644 --- a/config/connect-distributed-quickstart.properties +++ b/config/connect-distributed-quickstart.properties @@ -25,15 +25,15 @@ offset.flush.interval.ms=10000 plugin.path=connectors/ -group.id=kafka-connect-splunk-hec-sink -config.storage.topic=__kafka-connect-splunk-task-configs +group.id=splunk-kafka-connect-hec-sink +config.storage.topic=__splunk-kafka-connect-task-configs config.storage.replication.factor=1 -offset.storage.topic=__kafka-connect-splunk-offsets +offset.storage.topic=__splunk-kafka-connect-offsets offset.storage.replication.factor=1 offset.storage.partitions=1 -status.storage.topic=__kafka-connect-splunk-statuses +status.storage.topic=__splunk-kafka-connect-statuses status.storage.replication.factor=1 status.storage.partitions=1 diff --git a/config/connect-distributed.properties b/config/connect-distributed.properties index 9b4f6f4a..a0e93de4 100644 --- a/config/connect-distributed.properties +++ b/config/connect-distributed.properties @@ -25,15 +25,15 @@ offset.flush.interval.ms=10000 plugin.path=connectors/ -group.id=kafka-connect-splunk-hec-sink -config.storage.topic=__kafka-connect-splunk-task-configs +group.id=splunk-kafka-connect-hec-sink +config.storage.topic=__splunk-kafka-connect-task-configs config.storage.replication.factor=3 -offset.storage.topic=__kafka-connect-splunk-offsets +offset.storage.topic=__splunk-kafka-connect-offsets offset.storage.replication.factor=3 offset.storage.partitions=25 -status.storage.topic=__kafka-connect-splunk-statuses +status.storage.topic=__splunk-kafka-connect-statuses status.storage.replication.factor=3 status.storage.partitions=5 diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml index 4a6b1b89..881e627c 100644 --- a/dependency-reduced-pom.xml +++ b/dependency-reduced-pom.xml @@ -2,9 +2,9 @@ 4.0.0 com.github.splunk.kafka.connect - kafka-connect-splunk - kafka-connect-splunk - dev-SNAPSHOT + splunk-kafka-connect + splunk-kafka-connect + v1.0.0-LAR diff --git a/pom.xml b/pom.xml index 8bc4efe7..5b362c67 100644 --- a/pom.xml +++ b/pom.xml @@ -5,9 +5,9 @@ 4.0.0 com.github.splunk.kafka.connect - kafka-connect-splunk - dev-SNAPSHOT - kafka-connect-splunk + splunk-kafka-connect + v1.0.0-LAR + splunk-kafka-connect UTF-8 diff --git a/kafka-connect-splunk.iml b/splunk-kafka-connect.iml similarity index 97% rename from kafka-connect-splunk.iml rename to splunk-kafka-connect.iml index cbd51810..f7bbfa20 100644 --- a/kafka-connect-splunk.iml +++ b/splunk-kafka-connect.iml @@ -7,6 +7,7 @@ + diff --git a/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java b/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java index 70865868..7fccdf55 100644 --- a/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java +++ b/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java @@ -34,12 +34,12 @@ public final class SplunkSinkConnector extends SinkConnector { @Override public void start(Map taskConfig) { this.taskConfig = taskConfig; - log.info("kafka-connect-splunk starts"); + log.info("splunk-kafka-connect starts"); } @Override public void stop() { - log.info("kafka-connect-splunk stops"); + log.info("splunk-kafka-connect stops"); } @Override @@ -48,7 +48,7 @@ public List> taskConfigs(int maxTasks) { for (int i = 0; i < maxTasks; i++) { tasks.add(taskConfig); } - log.info("kafka-connect-splunk discovered {} tasks", tasks.size()); + log.info("splunk-kafka-connect discovered {} tasks", tasks.size()); return tasks; } diff --git a/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java b/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java index 8a768686..724bd11c 100644 --- a/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java +++ b/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java @@ -48,7 +48,7 @@ public void start(Map taskConfig) { tracker = new KafkaRecordTracker(); bufferedRecords = new ArrayList<>(); - log.info("kafka-connect-splunk task starts with config={}", connectorConfig); + log.info("splunk-kafka-connect task starts with config={}", connectorConfig); } @Override @@ -225,7 +225,7 @@ public void stop() { if (hec != null) { hec.close(); } - log.info("kafka-connect-splunk task ends with config={}", connectorConfig); + log.info("splunk-kafka-connect task ends with config={}", connectorConfig); } @Override diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7e7f1937..d6102e61 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -1,3 +1,3 @@ -githash=@8190911 -gitbranch=develop -gitversion=dev +githash=@e24583a +gitbranch=issue117-rename-directories +gitversion=v1.0.0-LAR From 0f3e74ee87211b0183ec2d59c1b718da6a7b35b0 Mon Sep 17 00:00:00 2001 From: gp510 Date: Tue, 13 Mar 2018 13:14:01 -0700 Subject: [PATCH 2/3] Updated naming per issue #117 --- README.md | 90 +++++++++---------- ci/Jenkinsfile | 2 +- ci/kafka_orca_gen.py | 2 +- ci/run_bastion.sh | 8 +- ci/run_kafka_connect.sh | 12 +-- .../connect-distributed-quickstart.properties | 8 +- config/connect-distributed.properties | 8 +- dependency-reduced-pom.xml | 4 +- ...ka-connect.iml => kafka-connect-splunk.iml | 1 - pom.xml | 4 +- .../kafka/connect/SplunkSinkConnector.java | 6 +- .../splunk/kafka/connect/SplunkSinkTask.java | 4 +- src/main/resources/version.properties | 2 +- 13 files changed, 75 insertions(+), 76 deletions(-) rename splunk-kafka-connect.iml => kafka-connect-splunk.iml (97%) diff --git a/README.md b/README.md index 7666c492..dbdb2cc3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## Kafka Connect Splunk +## Splunk Connect for Kafka A Kafka Connect Sink for Splunk features: @@ -17,17 +17,17 @@ A Kafka Connect Sink for Splunk features: ## Build -1. Clone the repo from https://github.com/splunk/splunk-kafka-connect +1. Clone the repo from https://github.com/splunk/kafka-connect-splunk 2. Verify that Java8 JRE or JDK is installed. -3. Run `bash build.sh`. The build script will download all dependencies and build the Splunk Kafka Connector. +3. Run `bash build.sh`. The build script will download all dependencies and build Splunk Connect for Kafka. -Note: The resulting "splunk-kafka-connect-*.tar.gz" package is self-contained. Bundled within it are the Kafka Connect framework, all 3rd party libraries, and the Splunk Kafka Connector. +Note: The resulting "splunk-kafka-connect*.tar.gz" package is self-contained. Bundled within it are the Kafka Connect framework, all 3rd party libraries, and Splunk Connect for Kafka. ## Quick Start 1. [Start](https://kafka.apache.org/quickstart) your Kafka Cluster and confirm it is running. 2. If this is a new install, create a test topic (eg: `perf`). Inject events into the topic. This can be done using [Kafka data-gen-app](https://github.com/dtregonning/kafka-data-gen) or the Kafka bundle [kafka-console-producer](https://kafka.apache.org/quickstart#quickstart_send). -3. Untar the package created from the build script: `tar xzvf splunk-kafka-connect-*.tar.gz` (Default target location is /tmp/splunk-kafka-connect-build/splunk-kafka-connect). +3. Untar the package created from the build script: `tar xzvf splunk-kafka-connect-*.tar.gz` (Default target location is /tmp/splunk-kafka-connect-build/kafka-connect-splunk). 4. Navigate to splunk-kafka-connect directory `cd splunk-kafka-connect`. 5. Adjust values for `bootstrap.servers` and `plugin.path` inside `config/connect-distributed-quickstart.properties` to fit your environment. Default values should work for experimentation. 6. Run `./bin/connect-distributed.sh config/connect-distributed-quickstart.properties` to start Kafka Connect. @@ -35,7 +35,7 @@ Note: The resulting "splunk-kafka-connect-*.tar.gz" package is self-contained. B ``` curl localhost:8083/connectors -X POST -H "Content-Type: application/json" -d '{ - "name": "splunk-kafka-connect", + "name": "kafka-connect-splunk", "config": { "connector.class": "com.splunk.kafka.connect.SplunkSinkConnector", "tasks.max": "3", @@ -71,32 +71,32 @@ Note: The resulting "splunk-kafka-connect-*.tar.gz" package is self-contained. B # List active connectors curl http://localhost:8083/connectors - # Get splunk-kafka-connect connector info - curl http://localhost:8083/connectors/splunk-kafka-connect + # Get kafka-connect-splunk connector info + curl http://localhost:8083/connectors/kafka-connect-splunk - # Get splunk-kafka-connect connector config info - curl http://localhost:8083/connectors/splunk-kafka-connect/config + # Get kafka-connect-splunk connector config info + curl http://localhost:8083/connectors/kafka-connect-splunk/config - # Delete splunk-kafka-connect connector - curl http://localhost:8083/connectors/splunk-kafka-connect -X DELETE + # Delete kafka-connect-splunk connector + curl http://localhost:8083/connectors/kafka-connect-splunk -X DELETE - # Get splunk-kafka-connect connector task info - curl http://localhost:8083/connectors/splunk-kafka-connect/tasks + # Get kafka-connect-splunk connector task info + curl http://localhost:8083/connectors/kafka-connect-splunk/tasks ``` See the [the Confluent doucumentation](https://docs.confluent.io/current/connect/managing.html#common-rest-examples) for additional REST examples. ## Deployment -Splunk Kafka Connector can run in containers, virtual machines or on physical machines. +Splunk Connect for Kafka can run in containers, virtual machines or on physical machines. You can leverage any automation tools for deployment. Use the following connector deployment options: -* Splunk Kafka Connector in a dedicated Kafka Connect Cluster (recommended) -* Splunk Kafka Connector in an existing Kafka Connect Cluster +* Splunk Connect for Kafka in a dedicated Kafka Connect Cluster (recommended) +* Splunk Connect for Kafka in an existing Kafka Connect Cluster ### Connector in a dedicated Kafka Connect Cluster -Running the Splunk Kafka Connector in a dedicated Kafka Connect Cluster is recommended. Isolating the Splunk connector from other Kafka connectors results in significant performance benefits in high throughput environments. +Running Splunk Connect for Kafka in a dedicated Kafka Connect Cluster is recommended. Isolating the Splunk connector from other Kafka connectors results in significant performance benefits in high throughput environments. 1. Untar the **splunk-kafka-connect-*.tar.gz** package and navigate to the **splunk-kafka-connect** directory. @@ -118,16 +118,16 @@ Running the Splunk Kafka Connector in a dedicated Kafka Connect Cluster is recom > Note: The below topics should be created by Kafka Connect when deploying the Splunk Connector. If the Kafka Connect cluster **does not have permission** to create these topics, create these manually before starting Kafka Connect cluster. ``` - group.id=splunk-kafka-connect-hec-sink # consumer group id of Kafka Connect, which is used to form a Kafka Connect cluster + group.id=kafka-connect-splunk-hec-sink # consumer group id of Kafka Connect, which is used to form a Kafka Connect cluster - config.storage.topic=__splunk-kafka-connect-task-configs # kafka topic used to persistent connector task configurations + config.storage.topic=__kafka-connect-splunk-task-configs # kafka topic used to persistent connector task configurations config.storage.replication.factor=3 - offset.storage.topic=__splunk-kafka-connect-offsets # kafka topic used to persistent task checkpoints + offset.storage.topic=__kafka-connect-splunk-offsets # kafka topic used to persistent task checkpoints offset.storage.replication.factor=3 offset.storage.partitions=25 - status.storage.topic=__splunk-kafka-connect-statuses # kafka topic used to persistent task statuses + status.storage.topic=__kafka-connect-splunk-statuses # kafka topic used to persistent task statuses status.storage.replication.factor=3 status.storage.partitions=5 ``` @@ -136,8 +136,8 @@ Running the Splunk Kafka Connector in a dedicated Kafka Connect Cluster is recom 5. Start Kafka Connect on all target hosts using the below commands: ``` - cd splunk-kafka-connect - export KAFKA_HEAP_OPTS="-Xmx6G -Xms2G" && ./bin/connect-distributed.sh config/connect-distributed.properties >> splunk-kafka-connect.log 2>&1 + cd kafka-connect-splunk + export KAFKA_HEAP_OPTS="-Xmx6G -Xms2G" && ./bin/connect-distributed.sh config/connect-distributed.properties >> kafka-connect-splunk.log 2>&1 ``` > Note: The **KAFKA\_HEAP\_OPTS** environment variable controls how much memory Kafka Connect can use. Set the **KAFKA\_HEAP\_OPTS** with the recommended value stated in the example above. @@ -167,13 +167,13 @@ internal.value.converter.schemas.enable=false offset.flush.interval.ms=10000 #Recommended -group.id=splunk-kafka-connect-hec-sink -config.storage.topic=__splunk-kafka-connect-task-configs +group.id=kafka-connect-splunk-hec-sink +config.storage.topic=__kafka-connect-splunk-task-configs config.storage.replication.factor=3 -offset.storage.topic=__splunk-kafka-connect-offsets +offset.storage.topic=__kafka-connect-splunk-offsets offset.storage.replication.factor=3 offset.storage.partitions=25 -status.storage.topic=__splunk-kafka-connect-statuses +status.storage.topic=__kafka-connect-splunk-statuses status.storage.replication.factor=3 status.storage.partitions=5 @@ -189,7 +189,7 @@ Please create or modify a Kafka Connect worker properties file to contain these 5. Validate your connector deployment by running the following command curl `http://:8083/connector-plugins`. Response should have an entry named `com.splunk.kafka.connect.SplunkSinkConnector`. ## Security -The Kafka Connect Splunk Sink supports the following security mechanisms: +Splunk Connect for Kafka supports the following security mechanisms: * `SSL` * `SASL/GSSAPI (Kerberos)` - starting at version 0.9.0.0 * `SASL/PLAIN` - starting at version 0.10.0.0 @@ -367,7 +367,7 @@ After Kafka Connect is brought up on every host, all of the Kafka Connect instan Even in a load balanced environment, a REST call can be executed against one of the cluster instances, and rest of the instances will pick up the task automatically. ### Configuration schema structure -Use the below schema to configure Splunk Kafka Connector +Use the below schema to configure Splunk Connect for Kafka ``` { @@ -406,7 +406,7 @@ Use the below schema to configure Splunk Kafka Connector * `name` - Connector name. A consumer group with this name will be created with tasks to be distributed evenly across the connector cluster nodes. * `connector.class` - The Java class used to perform connector jobs. Keep the default value **com.splunk.kafka.connect.SplunkSinkConnector** unless you modify the connector. -* `tasks.max` - The number of tasks generated to handle data collection jobs in parallel. The tasks will be spread evenly across all Splunk Kafka Connector nodes. +* `tasks.max` - The number of tasks generated to handle data collection jobs in parallel. The tasks will be spread evenly across all Splunk Connect for Kafka nodes. * `splunk.hec.uri` - Splunk HEC URIs. Either a list of FQDNs or IPs of all Splunk indexers, separated with a ",", or a load balancer. The connector will load balance to indexers using round robin. Splunk Connector will round robin to this list of indexers. ```https://hec1.splunk.com:8088,https://hec2.splunk.com:8088,https://hec3.splunk.com:8088``` * `splunk.hec.token` - [Splunk Http Event Collector token] (http://docs.splunk.com/Documentation/SplunkCloud/6.6.3/Data/UsetheHTTPEventCollector#About_Event_Collector_tokens). @@ -428,8 +428,8 @@ Use the below schema to configure Splunk Kafka Connector ### Acknowledgement Parameters #### Use Ack -* `splunk.hec.ack.enabled` - Valid settings are `true` or `false`. When set to `true` the Splunk Kafka Connector will poll event ACKs for POST events before check-pointing the Kafka offsets. This is used to prevent data loss, as this setting implements guaranteed delivery. By default, this setting is set to `true`. - > Note: If this setting is set to `true`, verify that the corresponding HEC token is also enabled with index acknowledgements, otherwise the data injection will fail, due to duplicate data. When set to `false`, the Splunk Kafka Connector will only POST events to your Splunk platform instance. After it receives a HTTP 200 OK response, it assumes the events are indexed by Splunk. Note: In cases where the Splunk platform crashes, there may be some data loss. +* `splunk.hec.ack.enabled` - Valid settings are `true` or `false`. When set to `true` Splunk Connect for Kafka will poll event ACKs for POST events before check-pointing the Kafka offsets. This is used to prevent data loss, as this setting implements guaranteed delivery. By default, this setting is set to `true`. + > Note: If this setting is set to `true`, verify that the corresponding HEC token is also enabled with index acknowledgements, otherwise the data injection will fail, due to duplicate data. When set to `false`, Splunk Connect for Kafka will only POST events to your Splunk platform instance. After it receives a HTTP 200 OK response, it assumes the events are indexed by Splunk. Note: In cases where the Splunk platform crashes, there may be some data loss. * `splunk.hec.ack.poll.interval` - This setting is only applicable when `splunk.hec.ack.enabled` is set to `true`. Internally it controls the event ACKs polling interval. By default, this setting is 10 seconds. * `splunk.hec.ack.poll.threads` - This setting is used for performance tuning and is only applicable when `splunk.hec.ack.enabled` is set to `true`. It controls how many threads should be spawned to poll event ACKs. By default, it is set to `1`. > Note: For large Splunk indexer clusters (For example, 100 indexers) you need to increase this number. Recommended increase to speed up ACK polling is 4 threads. @@ -440,7 +440,7 @@ Use the below schema to configure Splunk Kafka Connector ##### /raw endpoint only * `splunk.hec.raw.line.breaker` - Only applicable to /raw HEC endpoint. The setting is used to specify a custom line breaker to help Splunk separate the events correctly. - > Note: For example, you can specify "#####" as a special line breaker. Internally, the Splunk Kafka Connector will append this line breaker to every Kafka record to form a clear event boundary. The connector performs data injection in batch mode. On the Splunk platform side, you can configure **props.conf** to set up line breaker for the sourcetypes. Then the Splunk software will correctly break events for data flowing through /raw HEC endpoint. For questions on how and when to specify line breaker, go to the FAQ section. By default, this setting is empty. + > Note: For example, you can specify "#####" as a special line breaker. Internally, Splunk Connect for Kafka will append this line breaker to every Kafka record to form a clear event boundary. The connector performs data injection in batch mode. On the Splunk platform side, you can configure **props.conf** to set up line breaker for the sourcetypes. Then the Splunk software will correctly break events for data flowing through /raw HEC endpoint. For questions on how and when to specify line breaker, go to the FAQ section. By default, this setting is empty. ##### /event endpoint only * `splunk.hec.json.event.enrichment` - Only applicable to /event HEC endpoint. This setting is used to enrich raw data with extra metadata fields. It contains a list of key value pairs separated by ",". The configured enrichment metadata will be indexed along with raw event data by Splunk software. Note: Data enrichment for /event HEC endpoint is only available in Splunk Enterprise 6.5 and above. By default, this setting is empty. See ([Documentation](http://dev.splunk.com/view/event-collector/SP-CAAAE8Y#indexedfield)) for more information. @@ -584,7 +584,7 @@ A common architecture will include a load balancer in front of your Splunk platf ## Benchmark Results -A single Splunk Kafka Connector can reach maximum indexed throughput of **32 MB/second** with the following testbed and raw HEC endpoint in use: +A single instance of Splunk Connect for Kafka can reach maximum indexed throughput of **32 MB/second** with the following testbed and raw HEC endpoint in use: Hardware specifications: @@ -597,7 +597,7 @@ Hardware specifications: ## Scaling out your environment -Before scaling the Splunk Kafka Connector tier, ensure the bottleneck is in the connector tier and not in another component. +Before scaling the Splunk Connect for Kafka tier, ensure the bottleneck is in the connector tier and not in another component. Scaling out options: @@ -609,20 +609,20 @@ Scaling out options: ## Data loss and latency monitoring -When creating a Splunk Kafka Connector using the REST API, `"splunk.hec.track.data": "true"` can be configured to allow data loss tracking and data collection latency monitoring. +When creating an instance of Splunk Connect for Kafka using the REST API, `"splunk.hec.track.data": "true"` can be configured to allow data loss tracking and data collection latency monitoring. This is accomplished by enriching the raw data with **offset, timestamp, partition, topic** metadata. ### Data Loss Tracking -The Splunk Kafka Connector uses offset to track data loss since offsets in a Kafka topic partition are sequential. If a gap is observed in the Splunk software, there is data loss. +Splunk Connect for Kafka uses offset to track data loss since offsets in a Kafka topic partition are sequential. If a gap is observed in the Splunk software, there is data loss. ### Data Latency Tracking -The Splunk Kafka Connector uses the timestamp of the record to track the time elapsed between the time a Kafka record was created and the time the record was indexed in Splunk. +Splunk Connect for Kafka uses the timestamp of the record to track the time elapsed between the time a Kafka record was created and the time the record was indexed in Splunk. > Note: This setting will only work in conjunction with /event HEC endpoint (`"splunk.hec.raw" : "false"`) ### Malformed data -If the raw data of the Kafka records is a JSON object but is not able to be marshaled, or if the raw data is in bytes but it is not UTF-8 encodable, the Splunk Kafka Connector considers these records malformed. It will log the exception with Kafka specific information (topic, partition, offset) for these records within the console, as well as the malformed records information will be indexed in Splunk. Users can search "type=malformed" within Splunk to return any malformed Kafka records encountered. +If the raw data of the Kafka records is a JSON object but is not able to be marshaled, or if the raw data is in bytes but it is not UTF-8 encodable, Splunk Connect for Kafka considers these records malformed. It will log the exception with Kafka specific information (topic, partition, offset) for these records within the console, as well as the malformed records information will be indexed in Splunk. Users can search "type=malformed" within Splunk to return any malformed Kafka records encountered. ## FAQ @@ -650,12 +650,12 @@ If the raw data of the Kafka records is a JSON object but is not able to be mars 4. How many tasks should I configure? - Do not create more tasks than the number of partitions. Generally speaking, creating 2 * CPU tasks per Splunk Kafka Connector is a safe estimate. - > Note: For example, assume there are 5 Kafka Connects running the Splunk Kafka Connector. Each host is 8 CPUs with 16 GB memory. And there are 200 partitions to collect data from. `max.tasks` will be: `max.tasks` = 2 * CPUs/host * Kafka Connect instances = 2 * 8 * 5 = 80 tasks. Alternatively, if there are only 60 partitions to consume from, then just set max.tasks to 60. Otherwise, the remaining 20 will be pending, doing nothing. + Do not create more tasks than the number of partitions. Generally speaking, creating 2 * CPU tasks per instance of Splunk Connect for Kafka is a safe estimate. + > Note: For example, assume there are 5 Kafka Connects running Splunk Connect for Kafka. Each host is 8 CPUs with 16 GB memory. And there are 200 partitions to collect data from. `max.tasks` will be: `max.tasks` = 2 * CPUs/host * Kafka Connect instances = 2 * 8 * 5 = 80 tasks. Alternatively, if there are only 60 partitions to consume from, then just set max.tasks to 60. Otherwise, the remaining 20 will be pending, doing nothing. 5. How many Kafka Connect instances should I deploy? - This is highly dependent on how much volume per day the Splunk Kafka Connector needs to index in Splunk. In general an 8 CPU, 16 GB memory machine, can potentially achieve 50 - 60 MB/s throughput from Kafka into Splunk if Splunk is sized correctly. + This is highly dependent on how much volume per day Splunk Connect for Kafka needs to index in Splunk. In general an 8 CPU, 16 GB memory machine, can potentially achieve 50 - 60 MB/s throughput from Kafka into Splunk if Splunk is sized correctly. 6. How can I track data loss and data collection latency? @@ -676,9 +676,9 @@ If the raw data of the Kafka records is a JSON object but is not able to be mars ## Troubleshooting -1. Append the **log4j.logger.com.splunk=DEBUG** to **config/connect-log4j.properties** file to enable more verbose logging for Splunk Kafka Connector. +1. Append the **log4j.logger.com.splunk=DEBUG** to **config/connect-log4j.properties** file to enable more verbose logging for Splunk Connect for Kafka. 2. Kafka connect encounters an "out of memory" error. Remember to export environment variable **KAFKA\_HEAP\_OPTS="-Xmx6G -Xms2G"**. Refer to the [Deployment](#deployment) section for more information. -3. Can't see any Connector information on third party UI. For example, Splunk Kafka Connector is not shown on Confluent Control Center. Make sure cross origin access is enabled for Kafka Connect. Append the following two lines to connect configuration, e.g. `connect-distributed.properties` or `connect-distributed-quickstart.properties` and then restart Kafka Connect. +3. Can't see any Connector information on third party UI. For example, Splunk Connect for Kafka is not shown on the Confluent Control Center. Make sure cross origin access is enabled for Kafka Connect. Append the following two lines to connect configuration, e.g. `connect-distributed.properties` or `connect-distributed-quickstart.properties` and then restart Kafka Connect. ``` access.control.allow.origin=* diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b3c8a0f1..4582d35d 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -8,7 +8,7 @@ def dockerReq = new DockerRequest(steps, env, [imageName: "repo.splunk.com/splunk/products/splact:1.0.9", userId: "10777", - repoName: "git@github.com:splunk/splunk-kafka-connect.git", + repoName: "git@github.com:splunk/kafka-connect-splunk.git", runner: "yarn", remotePath: "/build"]) diff --git a/ci/kafka_orca_gen.py b/ci/kafka_orca_gen.py index 5099ab0d..2e9c00b4 100644 --- a/ci/kafka_orca_gen.py +++ b/ci/kafka_orca_gen.py @@ -6,7 +6,7 @@ DATA_GEN_IMAGE = 'repo.splunk.com/kafka-data-gen:0.4' KAFKA_IMAGE = 'repo.splunk.com/kafka-cluster:0.12' -KAFKA_CONNECT_IMAGE = 'repo.splunk.com/splunk-kafka-connect:1.8' +KAFKA_CONNECT_IMAGE = 'repo.splunk.com/kafka-connect-splunk:1.8' KAFKA_BASTION_IMAGE = 'repo.splunk.com/kafka-bastion:1.8' diff --git a/ci/run_bastion.sh b/ci/run_bastion.sh index 76b2279d..3f07507b 100755 --- a/ci/run_bastion.sh +++ b/ci/run_bastion.sh @@ -1,15 +1,15 @@ #!/bin/bash curdir=`pwd` -git clone git@github.com:splunk/splunk-kafka-connect.git +git clone git@github.com:splunk/kafka-connect-splunk.git branch=${KAFKA_CONNECT_BRANCH:-develop} -cd splunk-kafka-connect && git checkout ${branch} +cd kafka-connect-splunk && git checkout ${branch} duration=${SLEEP:-600} sleep ${duration} -bash ${curdir}/splunk-kafka-connect/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & +bash ${curdir}/kafka-connect-splunk/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & -python ${curdir}/splunk-kafka-connect/ci/perf.py +python ${curdir}/kafka-connect-splunk/ci/perf.py tail -f /dev/null diff --git a/ci/run_kafka_connect.sh b/ci/run_kafka_connect.sh index 8472140d..8f62593b 100755 --- a/ci/run_kafka_connect.sh +++ b/ci/run_kafka_connect.sh @@ -1,17 +1,17 @@ #!/bin/bash -# Checkout, build and run splunk-kafka-connect in the fight +# Checkout, build and run kafka-connect-splunk in the fight curdir=`pwd` -git clone git@github.com:splunk/splunk-kafka-connect.git +git clone git@github.com:splunk/kafka-connect-splunk.git branch=${KAFKA_CONNECT_BRANCH:-develop} # build the package -cd splunk-kafka-connect && git checkout ${branch} && bash build.sh +cd kafka-connect-splunk && git checkout ${branch} && bash build.sh # untar the package -tar xzf splunk-kafka-connect*.tar.gz -cd splunk-kafka-connect +tar xzf kafka-connect-splunk*.tar.gz +cd kafka-connect-splunk sed -i"" "s@bootstrap.servers=.*@bootstrap.servers=$KAFKA_BOOTSTRAP_SERVERS@g" config/connect-distributed.properties @@ -24,7 +24,7 @@ duration=${SLEEP:-300} sleep ${duration} echo "Run fix hosts" -bash ${curdir}/splunk-kafka-connect/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & +bash ${curdir}/kafka-connect-splunk/ci/fix_hosts.sh > /tmp/fixhosts 2>&1 & echo "Run proc monitor" cd proc_monitor diff --git a/config/connect-distributed-quickstart.properties b/config/connect-distributed-quickstart.properties index d05ad598..d11e119e 100644 --- a/config/connect-distributed-quickstart.properties +++ b/config/connect-distributed-quickstart.properties @@ -25,15 +25,15 @@ offset.flush.interval.ms=10000 plugin.path=connectors/ -group.id=splunk-kafka-connect-hec-sink -config.storage.topic=__splunk-kafka-connect-task-configs +group.id=kafka-connect-splunk-hec-sink +config.storage.topic=__kafka-connect-splunk-task-configs config.storage.replication.factor=1 -offset.storage.topic=__splunk-kafka-connect-offsets +offset.storage.topic=__kafka-connect-splunk-offsets offset.storage.replication.factor=1 offset.storage.partitions=1 -status.storage.topic=__splunk-kafka-connect-statuses +status.storage.topic=__kafka-connect-splunk-statuses status.storage.replication.factor=1 status.storage.partitions=1 diff --git a/config/connect-distributed.properties b/config/connect-distributed.properties index a0e93de4..9b4f6f4a 100644 --- a/config/connect-distributed.properties +++ b/config/connect-distributed.properties @@ -25,15 +25,15 @@ offset.flush.interval.ms=10000 plugin.path=connectors/ -group.id=splunk-kafka-connect-hec-sink -config.storage.topic=__splunk-kafka-connect-task-configs +group.id=kafka-connect-splunk-hec-sink +config.storage.topic=__kafka-connect-splunk-task-configs config.storage.replication.factor=3 -offset.storage.topic=__splunk-kafka-connect-offsets +offset.storage.topic=__kafka-connect-splunk-offsets offset.storage.replication.factor=3 offset.storage.partitions=25 -status.storage.topic=__splunk-kafka-connect-statuses +status.storage.topic=__kafka-connect-splunk-statuses status.storage.replication.factor=3 status.storage.partitions=5 diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml index 881e627c..1883c16a 100644 --- a/dependency-reduced-pom.xml +++ b/dependency-reduced-pom.xml @@ -2,8 +2,8 @@ 4.0.0 com.github.splunk.kafka.connect - splunk-kafka-connect - splunk-kafka-connect + kafka-connect-splunk + kafka-connect-splunk v1.0.0-LAR diff --git a/splunk-kafka-connect.iml b/kafka-connect-splunk.iml similarity index 97% rename from splunk-kafka-connect.iml rename to kafka-connect-splunk.iml index f7bbfa20..cbd51810 100644 --- a/splunk-kafka-connect.iml +++ b/kafka-connect-splunk.iml @@ -7,7 +7,6 @@ - diff --git a/pom.xml b/pom.xml index 5b362c67..1bf7ea0c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,9 +5,9 @@ 4.0.0 com.github.splunk.kafka.connect - splunk-kafka-connect + kafka-connect-splunk v1.0.0-LAR - splunk-kafka-connect + kafka-connect-splunk UTF-8 diff --git a/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java b/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java index 7fccdf55..70865868 100644 --- a/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java +++ b/src/main/java/com/splunk/kafka/connect/SplunkSinkConnector.java @@ -34,12 +34,12 @@ public final class SplunkSinkConnector extends SinkConnector { @Override public void start(Map taskConfig) { this.taskConfig = taskConfig; - log.info("splunk-kafka-connect starts"); + log.info("kafka-connect-splunk starts"); } @Override public void stop() { - log.info("splunk-kafka-connect stops"); + log.info("kafka-connect-splunk stops"); } @Override @@ -48,7 +48,7 @@ public List> taskConfigs(int maxTasks) { for (int i = 0; i < maxTasks; i++) { tasks.add(taskConfig); } - log.info("splunk-kafka-connect discovered {} tasks", tasks.size()); + log.info("kafka-connect-splunk discovered {} tasks", tasks.size()); return tasks; } diff --git a/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java b/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java index 724bd11c..8a768686 100644 --- a/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java +++ b/src/main/java/com/splunk/kafka/connect/SplunkSinkTask.java @@ -48,7 +48,7 @@ public void start(Map taskConfig) { tracker = new KafkaRecordTracker(); bufferedRecords = new ArrayList<>(); - log.info("splunk-kafka-connect task starts with config={}", connectorConfig); + log.info("kafka-connect-splunk task starts with config={}", connectorConfig); } @Override @@ -225,7 +225,7 @@ public void stop() { if (hec != null) { hec.close(); } - log.info("splunk-kafka-connect task ends with config={}", connectorConfig); + log.info("kafka-connect-splunk task ends with config={}", connectorConfig); } @Override diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d6102e61..109dbd66 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -1,3 +1,3 @@ -githash=@e24583a +githash=@ae68d5b gitbranch=issue117-rename-directories gitversion=v1.0.0-LAR From e9d2944e4577e48c475abe566f33370be236de0f Mon Sep 17 00:00:00 2001 From: gp510 Date: Thu, 15 Mar 2018 11:00:57 -0700 Subject: [PATCH 3/3] Updates to readme and build properties per issue #117 --- README.md | 4 ++-- dependency-reduced-pom.xml | 4 ++-- pom.xml | 4 ++-- src/main/resources/version.properties | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dbdb2cc3..c913da6e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Splunk Connect for Kafka -A Kafka Connect Sink for Splunk features: +Splunk Connect for Kafka is a Kafka Connect Sink for Splunk with the following features: * Data ingestion from Kafka topics into Splunk via [Splunk HTTP Event Collector(HEC)](http://dev.splunk.com/view/event-collector/SP-CAAAE6M). * In-flight data transformation and enrichment. @@ -144,7 +144,7 @@ Running Splunk Connect for Kafka in a dedicated Kafka Connect Cluster is recomme ### Connector in an existing Kafka Connect Cluster -1. Navigate to Splunkbase and download the latest version of [Splunk Kafka Connect](https://splunkbase.splunk.com/app/3862/). +1. Navigate to Splunkbase and download the latest version of [Splunk Connect for Kafka](https://splunkbase.splunk.com/app/3862/). 2. Copy downloaded file onto every host running Kafka Connect into the directory that contains your other connectors or create a folder to store them in. (ex. `/opt/connectors/splunk-kafka-connect`) diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml index 1883c16a..881e627c 100644 --- a/dependency-reduced-pom.xml +++ b/dependency-reduced-pom.xml @@ -2,8 +2,8 @@ 4.0.0 com.github.splunk.kafka.connect - kafka-connect-splunk - kafka-connect-splunk + splunk-kafka-connect + splunk-kafka-connect v1.0.0-LAR diff --git a/pom.xml b/pom.xml index 1bf7ea0c..5b362c67 100644 --- a/pom.xml +++ b/pom.xml @@ -5,9 +5,9 @@ 4.0.0 com.github.splunk.kafka.connect - kafka-connect-splunk + splunk-kafka-connect v1.0.0-LAR - kafka-connect-splunk + splunk-kafka-connect UTF-8 diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 109dbd66..f306651a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -1,3 +1,3 @@ -githash=@ae68d5b +githash=@0f3e74e gitbranch=issue117-rename-directories gitversion=v1.0.0-LAR