Support for DataFlow Deployments

docs/configuration.yml

@@ -33,6 +33,37 @@ df:
     persist:
   force_delete:
   terminate_deployments:
+  tags:
+  deployments:
+    - name:
+      flow_ver_crn:
+      size:
+      static_node_count:
+      autoscale:
+      autoscale_nodes_min:
+      autoscale_nodes_max:
+      nifi_ver:
+      autostart_flow:
+      parameter_groups:
+        - name:
+          parameters:
+            - name:
+              value:
+              assetReferences: [str, ...]
+      kpis:
+        - metricId:
+          componentId:
+          alert:
+            thresholdMoreThan:
+              unitId:
+              value:
+            thresholdLessThan:
+              unitId:
+              value:
+            frequencyTolerance:
+              value:
+                unit:
+                  id: SECONDS|MINUTES|HOURS|DAYS
 dw:
   definitions:
   suffix:

roles/common/defaults/main.yml

@@ -61,6 +61,7 @@ common__cdp_control_plane_region:         "{{ globals.cdp_region | default('us-w
 common__cdp_control_plane_crn:            "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}"
 # Infra
 common__infra_deployment_engine:          "{{ globals.infra_deployment_engine | default('ansible') }}"
+common__aws_profile:                      "{{ globals.aws_profile | default('') }}"
 common__infra_type:                       "{{ globals.infra_type | default('aws') }}"
 common__public_key_file:                  "{{ globals.ssh.public_key_file | default('') }}"
 common__namespace_cdp:                    "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}"
@@ -74,15 +75,15 @@ common__storage_name:                     "{{ infra.storage.name | default([comm
 common__terraform_base_dir:               "{{ globals.terraform_base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}"
 # The processed Jinja template files for Terraform are placed in common__terraform_template_dir
 common__terraform_template_dir:            "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}"
-# A timestamped artefact directory storing a copy of the Terraform code from each run 
+# A timestamped artefact directory storing a copy of the Terraform code from each run
 common__terraform_artefact_dir:            "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}"
 # Terraform apply/destroy run from under this directory
 common__terraform_workspace_dir:           "{{ [common__terraform_base_dir, 'workspace'] | path_join }}"
 
 common__terraform_allowed_state_storage:   "['local', 'remote_s3']"
 common__terraform_state_storage:           "{{ globals.terraform_state_storage  | default('local') }}"
-common__terraform_remote_state_bucket:     "{{ globals.terraform_remote_state_bucket | default('') }}"      
-common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}"      
+common__terraform_remote_state_bucket:     "{{ globals.terraform_remote_state_bucket | default('') }}"
+common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}"
 
 common__vpc_name:                         "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}"
 common__vpc_public_subnet_cidrs:          "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}"
@@ -109,7 +110,6 @@ common__aws_vpc_id:                       "{{ infra.aws.vpc.existing.vpc_id | de
 common__aws_public_subnet_ids:            "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}"
 common__aws_private_subnet_ids:           "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
 common__aws_region:                       "{{ infra.aws.region | default('eu-west-1') }}"
-common__aws_profile:                      "{{ infra.aws.profile | default('') }}"
 common__aws_role_suffix:                  "{{ infra.aws.role.suffix | default(common__role_suffix) }}"
 common__aws_datalake_admin_role_name:     "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}"
 common__aws_datalake_admin_suffix:        "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}"

roles/info/tasks/main.yml

@@ -80,13 +80,19 @@
     env:  "{{ info__env_name }}"
   register: __opdb_info
 
+- name: Query CDP DFX
+  cloudera.cloud.df_service_info:
+    name:  "{{ info__env_name }}"
+  register: __df_info
+
 - name: Set facts for the CDP deployment details
   ansible.builtin.set_fact:
     deployment:
       environment: "{{ __env_info.environments | first | default({}) }}"
       datalake: "{{ __datalake_info.datalakes | first | default({}) }}"
       datahubs: "{{ __datahubs_info.datahubs }}"
       workspaces: "{{ __ml_info.workspaces }}"
+      dataflow: "{{ __df_info.services }}"
       services: "{{ __de_info.services }}"
       operational_dbs: "{{ __opdb_info.databases }}"
 

roles/runtime/defaults/main.yml

@@ -83,7 +83,11 @@ run__df_cluster_subnets:            "{{ df.cluster_subnets | default([]) }}"
 run__df_lb_subnets:                 "{{ df.loadbalancer_subnets | default([]) }}"
 run__df_persist:                    "{{ df.teardown.persist | default(False) }}"
 run__df_force_delete:               "{{ df.force_delete | default(run__force_teardown) }}"
-run__df_terminate_deployments:      "{{ df.terminate_deployments | default(run__force_teardown) }}"
+run__df_terminate_deployments:      "{{ df.terminate_deployments | default(True) }}"
+run__df_tags:                       "{{ df.tags | default(common__tags) }}"
+run__df_deployments:                "{{ df.deployments | default([]) }}"
+run__df_readyflows:                 "{{ df.readyflows | default([]) }}"
+run__df_delete_readyflows:          "{{ df.delete_imported_readyflows | default(False) }}"
 
 # Deploy
 run__include_ml:                     "{{ common__include_ml }}"

roles/runtime/tasks/initialize_base.yml

@@ -166,6 +166,7 @@
         label: "{{ config.name }}"
 
 - name: Prepare for CDP DE Service experiences
+  tags: de
   when: run__include_de
   block:
     - name: Construct CDP DE Service configurations
@@ -181,3 +182,22 @@
       loop_control:
         loop_var: __de_config
         label: "{{ config.name }}"
+
+- name: Prepare for CDP DF Service experiences
+  tags: df
+  when:
+    - run__include_df
+    - run__df_deployments | length > 0
+  block:
+    - name: Construct CDP DF Deployment configurations
+      ansible.builtin.set_fact:
+        run__df_configs: "{{ run__df_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __df_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        config:
+          name: "{{ __df_config.name | default([run__namespace_cdp, __df_config.flow_name[::2] | replace(' ','') ] | join('-')) }}"
+          raw: "{{ __df_config }}"
+      loop: "{{ run__df_deployments }}"
+      loop_control:
+        loop_var: __df_config
+        label: "{{ config.name }}"

roles/runtime/tasks/initialize_setup.yml

@@ -29,3 +29,5 @@
     - dw
     - opdb
     - dh
+    - df
+    - de

roles/runtime/tasks/initialize_teardown.yml

@@ -27,7 +27,7 @@
 - name: Discover CDP DF Deployments
   register: run__df_service_info
   when: run__include_df
-  cloudera.cloud.df_info:
+  cloudera.cloud.df_service_info:
     name: "{{ run__env_name }}"
 
 - name: Initialize Purge of all Runtimes in Environment

roles/runtime/tasks/setup_base.yml

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Request Service Deployments
 - name: Request CDP Datahub deployments
   when: run__include_datahub
   tags: dh
@@ -63,6 +64,7 @@
 
 - name: Execute CDP DE Service experiences setup
   when: run__include_de
+  tags: de
   cloudera.cloud.de:
     name: "{{ __de_config_item.name }}"
     env: "{{ run__env_name }}"
@@ -117,9 +119,11 @@
     kube_ip_ranges: "{{ run__df_kube_ip_ranges }}"
     cluster_subnets: "{{ run__df_cluster_subnets }}"
     loadbalancer_subnets: "{{ run__df_lb_subnets }}"
+    tags: "{{ run__df_tags }}"
     state: present
     wait: no
 
+# Wait for Service Deployments
 - name: Wait for CDP Datahub deployments to complete
   when: run__include_datahub
   tags: dh
@@ -150,6 +154,7 @@
 
 - name: Wait for CDP DE Service experiences to complete
   when: run__include_de
+  tags: de
   ansible.builtin.async_status:
     jid: "{{ __de_build.ansible_job_id }}"
   loop_control:
@@ -175,15 +180,17 @@
   retries: 120
   delay: 30
 
-- name: Wait for CDP Dataflow deployment to complete
+- name: Wait for CDP Dataflow Service Enablement to complete
   when: run__include_df
   tags: df
   cloudera.cloud.df_service:
     env_crn: "{{ run__cdp_env_crn }}"
     wait: yes
 
+# Request Service child deployments
 - name: Create CDP DE Virtual clusters
   when: run__include_de
+  tags: de
   cloudera.cloud.de_virtual_cluster:
     cluster_name: "{{ __de_vc_config_item.0.name }}"
     env: "{{ run__env_name }}"
@@ -206,8 +213,48 @@
     index_var: __de_vc_index
     label: "{{ __de_vc_config_item.0.name | default ('None') }}"
 
+- name: Ensure requested CDP DF ReadyFlows are imported
+  when: run__include_df
+  tags: df
+  cloudera.cloud.df_readyflow:
+    name: "{{ __df_readyflow_item.flow_name }}"
+  loop: "{{ run__df_readyflows }}"
+  loop_control:
+    loop_var: __df_readyflow_item
+    label: "{{ __df_readyflow_item.flow_name }}"
+
+- name: Create CDP DF Deployments
+  when:
+    - run__include_df
+    - run__df_configs is defined
+  tags: df
+  cloudera.cloud.df_deployment:
+    name: "{{ __df_deploy_item.name }}"
+    df_name: "{{ run__env_name }}"
+    flow_ver_crn: "{{ __df_deploy_item.raw.flow_ver_crn | default(omit) }}"
+    flow_name: "{{ __df_deploy_item.raw.flow_name | default(omit) }}"
+    flow_ver: "{{ __df_deploy_item.raw.flow_ver | default(omit) }}"
+    size: "{{ __df_deploy_item.raw.size | default(omit) }}"
+    static_node_count: "{{ __df_deploy_item.raw.static_node_count | default(omit) }}"
+    autoscale: "{{ __df_deploy_item.raw.autoscale | default(omit) }}"
+    autoscale_nodes_min: "{{ __df_deploy_item.raw.autoscale_nodes_min | default(omit) }}"
+    autoscale_nodes_max: "{{ __df_deploy_item.raw.autoscale_nodes_max | default(omit) }}"
+    nifi_ver: "{{ __df_deploy_item.raw.nifi_ver | default(omit) }}"
+    autostart_flow: "{{ __df_deploy_item.raw.autostart_flow | default(omit) }}"
+    parameter_groups: "{{ __df_deploy_item.raw.parameter_groups | default(omit) }}"
+    kpis: "{{ __df_deploy_item.raw.kpis | default(omit) }}"
+  async: 720
+  poll: 0
+  register: __df_deployments
+  loop: "{{ run__df_configs }}"
+  loop_control:
+    loop_var: __df_deploy_item
+    label: "{{ __df_deploy_item.name }}"
+
+# Wait for Service child deployments
 - name: Wait for CDP DE Virtual cluster setup to complete
   when: run__include_de
+  tags: de
   ansible.builtin.async_status:
     jid: "{{ __de_vc_build.ansible_job_id }}"
   register: __de_vc_builds_async
@@ -218,3 +265,19 @@
   loop_control:
     loop_var: __de_vc_build
     label: "{{ __de_vc_build.__de_vc_config_item.0.name | default ('None') }}"
+
+- name: Wait for CDP DF Deployments to complete
+  when:
+    - run__include_df
+    - run__df_configs is defined
+  tags: df
+  ansible.builtin.async_status:
+    jid: "{{ __df_deployment.ansible_job_id }}"
+  register: __df_deployments_async
+  until: __df_deployments_async.finished
+  retries: 60
+  delay: 15
+  loop: "{{ __df_deployments.results }}"
+  loop_control:
+    loop_var: __df_deployment
+    label: "{{ __df_deployment.__df_deploy_item.0.name | default ('None') }}"

roles/runtime/tasks/teardown_base.yml

@@ -190,16 +190,30 @@
   retries: 120
   delay: 30
 
-- name: Wait for CDP Dataflow deployment to decommission
+- name: Wait for CDP Dataflow Service to decommission
   when:
     - run__include_df
     - run__df_service_info.services | length > 0
   cloudera.cloud.df_service:
     df_crn: "{{ __df_teardown_wait_item.crn }}"
     persist: "{{ run__df_persist }}"
     force: "{{ run__df_force_delete }}"
+    terminate: "{{ run__df_terminate_deployments }}"
     state: absent
     wait: yes
   loop_control:
     loop_var: __df_teardown_wait_item
   loop: "{{ run__df_service_info.services }}"
+
+- name: Ensure requested CDP DF ReadyFlow imports are deleted from Tenant
+  when:
+    - run__include_df
+    - run__df_delete_readyflows | bool
+  tags: df
+  cloudera.cloud.df_readyflow:
+    name: "{{ __df_readyflow_item.flow_name }}"
+    state: absent
+  loop: "{{ run__df_readyflows }}"
+  loop_control:
+    loop_var: __df_readyflow_item
+    label: "{{ __df_readyflow_item.flow_name }}"