Merge pull request #253 from splunk/INGEST-17730

Kafka CI pipeline for multiple version of kafka

Author: chaitanyaphalak

.circleci/config.yml

@@ -1,38 +1,76 @@
 version: 2.1
 
 workflows:
-  version: 2.1
   test:
     jobs:
-      - build_test:
+#      - build_test_kafka:
+#          kafka_version: 1.0.0
+#          kafka_package: kafka_2.11-1.0.0.tgz
+#          filters:
+#            branches:
+#              ignore:
+#                - /^release\/.*/
+#                - master
+      - build_test_kafka:
+          kafka_version: 1.1.1
+          kafka_package: kafka_2.11-1.1.1.tgz
+          filters:
+            branches:
+              ignore:
+                - /^release\/.*/
+                - master
+      - build_test_kafka:
+          kafka_version: 2.0.0
+          kafka_package: kafka_2.11-2.0.0.tgz
+          filters:
+            branches:
+              ignore:
+                - /^release\/.*/
+                - master
+      - build_test_kafka:
+          kafka_version: 2.1.0
+          kafka_package: kafka_2.12-2.1.0.tgz
+          filters:
+            branches:
+              ignore:
+                - /^release\/.*/
+                - master
+      - build_test_kafka:
+          kafka_version: 2.6.0
+          kafka_package: kafka_2.13-2.6.0.tgz
           filters:
             branches:
               ignore:
                 - /^release\/.*/
                 - master
 
 jobs:
-  build_test:
-    resource_class: large
-    machine:
-      image: ubuntu-1604:201903-01
-    environment:
-      MINIKUBE_HOME: /home/circleci
-      CI_SPLUNK_VERSION: "8.0.2.1"
-      CI_SPLUNK_FILENAME: splunk-8.0.2.1-f002026bad55-linux-2.6-amd64.deb
-      CI_SPLUNK_HOST: 127.0.0.1
-      CI_SPLUNK_PORT: 8089
-      CI_SPLUNK_USERNAME: admin
-      CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
-      CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
-      CI_SPLUNK_PASSWORD: helloworld
-      CI_INDEX_EVENTS: main
-      CI_KAFKA_HEADER_INDEX: kafka
-      CI_DATAGEN_IMAGE: rock1017/log-generator:latest
-      CI_KAFKA_VERSION: 2.4.0
-      CI_KAFKA_FILENAME: kafka_2.13-2.4.0.tgz
-      CI_OLD_CONNECTOR_VERSION: v1.2.0
+  build_test_kafka:
     working_directory: ~/repo
+    executor:
+      name: kafka_connect_linux
+    parameters:
+      kafka_version:
+        description: "Kafka version"
+        type: string
+      kafka_package:
+        description: "Kafka package name"
+        type: string
+    steps:
+      - build_test:
+          kafka_version: << parameters.kafka_version >>
+          kafka_package: << parameters.kafka_package >>
+
+
+commands:
+  build_test:
+    parameters:
+      kafka_version:
+        description: "Kafka version"
+        type: string
+      kafka_package:
+        description: "Kafka package name"
+        type: string
     steps:
       - attach_workspace:
           at: /tmp
@@ -59,15 +97,6 @@ jobs:
       - run:
           name: Create file for checksum
           command: echo $CI_SPLUNK_VERSION > /tmp/splunk_version.txt
-      - install_splunk
-      - install_kafka
-      - test_kafka_connect_upgrade
-      - run_functional_tests
-
-commands:
-  install_splunk:
-    description: "Install Splunk"
-    steps:
       - run:
           name: Install Splunk
           command: |
@@ -109,16 +138,12 @@ commands:
             curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_INDEX_EVENTS&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
             curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_KAFKA_HEADER_INDEX&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
             sudo /opt/splunk/bin/splunk restart --accept-license --answer-yes --no-prompt
-
-  install_kafka:
-    description: "Install Kafka"
-    steps:
       - run:
-          name: Install Kafka
+          name: Install Kafka << parameters.kafka_version >>
           command: |
-            cd /tmp && wget http://www-us.apache.org/dist/kafka/$CI_KAFKA_VERSION/$CI_KAFKA_FILENAME
-            sudo tar xzf $CI_KAFKA_FILENAME
-            rm $CI_KAFKA_FILENAME
+            cd /tmp && wget https://archive.apache.org/dist/kafka/<< parameters.kafka_version >>/<< parameters.kafka_package >>
+            sudo tar xzf << parameters.kafka_package >>
+            rm << parameters.kafka_package >>
             sudo mv kafka_* /usr/local/kafka
             sudo tee /etc/systemd/system/zookeeper.service > /dev/null \<< EOF
             [Unit]
@@ -154,9 +179,6 @@ commands:
             sudo systemctl daemon-reload
             sudo systemctl start zookeeper
             sudo systemctl start kafka
-  test_kafka_connect_upgrade:
-    description: "Test kafka connect upgrade"
-    steps:
       - run:
           name: Test kafka connect upgrade
           command: |
@@ -173,10 +195,6 @@ commands:
             export PYTHONWARNINGS="ignore:Unverified HTTPS request"
             echo "Test kafka connect upgrade ..."
             python test/lib/connector_upgrade.py
-
-  run_functional_tests:
-    description: "Run functional tests on current kafka connect version"
-    steps:
       - run:
           name: Install kafka connect
           command: |
@@ -191,3 +209,23 @@ commands:
             sleep 5
             echo "Running functional tests....."
             python -m pytest -p no:warnings -s
+
+executors:
+  kafka_connect_linux:
+    resource_class: large
+    machine:
+      image: ubuntu-1604:201903-01
+    environment:
+      MINIKUBE_HOME: /home/circleci
+      CI_SPLUNK_VERSION: "8.0.2.1"
+      CI_SPLUNK_FILENAME: splunk-8.0.2.1-f002026bad55-linux-2.6-amd64.deb
+      CI_SPLUNK_HOST: 127.0.0.1
+      CI_SPLUNK_PORT: 8089
+      CI_SPLUNK_USERNAME: admin
+      CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
+      CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
+      CI_SPLUNK_PASSWORD: helloworld
+      CI_INDEX_EVENTS: main
+      CI_KAFKA_HEADER_INDEX: kafka
+      CI_DATAGEN_IMAGE: rock1017/log-generator:latest
+      CI_OLD_CONNECTOR_VERSION: v1.2.0

test/lib/commonkafka.py

@@ -118,15 +118,18 @@ def get_running_kafka_connector_task_status(setup, params):
     '''
     Get running kafka connect connector tasks status using kafka connect REST API
     '''
-    response = requests.get(url=setup["kafka_connect_url"] + "/connectors/" + params["name"] + "/status",
-                            headers={'Accept': 'application/json', 'Content-Type': 'application/json'})
-    content = response.json()
-
-    if content.get('connector'):
-        if content['connector']['state'] == 'RUNNING':
-            task_status = jsonpath.jsonpath(content, '$.tasks.*.state')
-            return task_status
-
+    t_end = time.time() + 60
+    url = setup["kafka_connect_url"] + "/connectors/" + params["name"] + "/status"
+    header = {'Accept': 'application/json', 'Content-Type': 'application/json'}
+    while time.time() < t_end:
+        content = requests.get(url=url, headers=header).json()
+        if content.get('connector'):
+            if content['connector']['state'] == 'RUNNING' and len(content['tasks']) > 0:
+                time.sleep(2)
+                content = requests.get(url=url, headers=header).json()
+                logger.info(content)
+                task_status = jsonpath.jsonpath(content, '$.tasks.*.state')
+                return task_status
 
 
 

test/testcases/test_crud.py

@@ -128,5 +128,4 @@ def test_invalid_crud_event_enrichment_tasks(self, setup, test_case, config_inpu
         setup['connectors'].append(test_case)
 
         assert create_kafka_connector(setup, connector_definition_invalid_tasks) is True
-        time.sleep(2)
         assert get_running_kafka_connector_task_status(setup, connector_definition_invalid_tasks) == expected