From 39dffb6e4dbdb5ad5afb3269f1a85fb8d462dd74 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Fri, 29 Oct 2021 18:32:15 +0100 Subject: [PATCH 1/5] WIP Signed-off-by: Daniel Chaffelson --- roles/runtime/tasks/initialize_teardown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index 2baaba83..5ec802a1 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -27,7 +27,7 @@ - name: Discover CDP DF Deployments register: run__df_service_info when: run__include_df - cloudera.cloud.df_info: + cloudera.cloud.df_service_info: name: "{{ run__env_name }}" - name: Initialize Purge of all Runtimes in Environment From 84cc1ea20bfb1e0318ee401d1794d17644de220c Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 16 Dec 2021 20:27:21 +0000 Subject: [PATCH 2/5] WIP Signed-off-by: Daniel Chaffelson --- docs/configuration.yml | 31 +++++++++++++++ roles/runtime/defaults/main.yml | 4 +- roles/runtime/tasks/initialize_base.yml | 20 ++++++++++ roles/runtime/tasks/initialize_setup.yml | 2 + roles/runtime/tasks/setup_base.yml | 49 ++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/configuration.yml b/docs/configuration.yml index 2400601f..5ce97a6d 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -33,6 +33,37 @@ df: persist: force_delete: terminate_deployments: + tags: + deployments: + - name: + flow_ver_crn: + size: + static_node_count: + autoscale: + autoscale_nodes_min: + autoscale_nodes_max: + nifi_ver: + autostart_flow: + parameter_groups: + - name: + parameters: + - name: + value: + assetReferences: [str, ...] + kpis: + - metricId: + componentId: + alert: + thresholdMoreThan: + unitId: + value: + thresholdLessThan: + unitId: + value: + frequencyTolerance: + value: + unit: + id: SECONDS|MINUTES|HOURS|DAYS dw: definitions: suffix: diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 765152fe..9734255f 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -83,7 +83,9 @@ run__df_cluster_subnets: "{{ df.cluster_subnets | default([]) }}" run__df_lb_subnets: "{{ df.loadbalancer_subnets | default([]) }}" run__df_persist: "{{ df.teardown.persist | default(False) }}" run__df_force_delete: "{{ df.force_delete | default(run__force_teardown) }}" -run__df_terminate_deployments: "{{ df.terminate_deployments | default(run__force_teardown) }}" +run__df_terminate_deployments: "{{ df.terminate_deployments | default(True) }}" +run__df_tags: "{{ df.tags | default(common__tags) }}" +run__df_deployments: "{{ df.deployments | default([]) }}" # Deploy run__include_ml: "{{ common__include_ml }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index b2d84d4f..06ccbc11 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -166,6 +166,7 @@ label: "{{ config.name }}" - name: Prepare for CDP DE Service experiences + tags: de when: run__include_de block: - name: Construct CDP DE Service configurations @@ -181,3 +182,22 @@ loop_control: loop_var: __de_config label: "{{ config.name }}" + +- name: Prepare for CDP DF Service experiences + tags: df + when: + - run__include_df + - run__df_deployments | length > 0 + block: + - name: Construct CDP DF Deployment configurations + ansible.builtin.set_fact: + run__df_configs: "{{ run__df_configs | default([]) | union([config]) }}" + vars: + include: "{{ lookup('template', __df_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" + config: + name: "{{ __df_config.name | default([run__namespace_cdp, __df_config.flow_name[::2] | replace(' ','') ] | join('-')) }}" + raw: "{{ __df_config }}" + loop: "{{ run__df_deployments }}" + loop_control: + loop_var: __df_config + label: "{{ config.name }}" diff --git a/roles/runtime/tasks/initialize_setup.yml b/roles/runtime/tasks/initialize_setup.yml index 40bc06a6..25d20c4c 100644 --- a/roles/runtime/tasks/initialize_setup.yml +++ b/roles/runtime/tasks/initialize_setup.yml @@ -29,3 +29,5 @@ - dw - opdb - dh + - df + - de diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index a1991f6d..3c564147 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Request Service Deployments - name: Request CDP Datahub deployments when: run__include_datahub tags: dh @@ -63,6 +64,7 @@ - name: Execute CDP DE Service experiences setup when: run__include_de + tags: de cloudera.cloud.de: name: "{{ __de_config_item.name }}" env: "{{ run__env_name }}" @@ -117,9 +119,11 @@ kube_ip_ranges: "{{ run__df_kube_ip_ranges }}" cluster_subnets: "{{ run__df_cluster_subnets }}" loadbalancer_subnets: "{{ run__df_lb_subnets }}" + tags: "{{ run__df_tags }}" state: present wait: no +# Wait for Service Deployments - name: Wait for CDP Datahub deployments to complete when: run__include_datahub tags: dh @@ -150,6 +154,7 @@ - name: Wait for CDP DE Service experiences to complete when: run__include_de + tags: de ansible.builtin.async_status: jid: "{{ __de_build.ansible_job_id }}" loop_control: @@ -182,8 +187,10 @@ env_crn: "{{ run__cdp_env_crn }}" wait: yes +# Request Service child deployments - name: Create CDP DE Virtual clusters when: run__include_de + tags: de cloudera.cloud.de_virtual_cluster: cluster_name: "{{ __de_vc_config_item.0.name }}" env: "{{ run__env_name }}" @@ -206,8 +213,36 @@ index_var: __de_vc_index label: "{{ __de_vc_config_item.0.name | default ('None') }}" +- name: Create CDP DF Deployments + when: run__include_df + tags: df + cloudera.cloud.df_deployment: + name: "{{ __df_deploy_item.name }}" + df_name: "{{ run__env_name }}" + flow_ver_crn: "{{ __df_deploy_item.raw.flow_ver_crn | default(omit) }}" + flow_name: "{{ __df_deploy_item.raw.flow_name | default(omit) }}" + flow_ver: "{{ __df_deploy_item.raw.flow_ver | default(omit) }}" + size: "{{ __df_deploy_item.raw.size | default(omit) }}" + static_node_count: "{{ __df_deploy_item.raw.static_node_count | default(omit) }}" + autoscale: "{{ __df_deploy_item.raw.autoscale | default(omit) }}" + autoscale_nodes_min: "{{ __df_deploy_item.raw.autoscale_nodes_min | default(omit) }}" + autoscale_nodes_max: "{{ __df_deploy_item.raw.autoscale_nodes_max | default(omit) }}" + nifi_ver: "{{ __df_deploy_item.raw.nifi_ver | default(omit) }}" + autostart_flow: "{{ __df_deploy_item.raw.autostart_flow | default(omit) }}" + parameter_groups: "{{ __df_deploy_item.raw.parameter_groups | default(omit) }}" + kpis: "{{ __df_deploy_item.raw.kpis | default(omit) }}" + async: 720 + poll: 0 + register: __df_deployments + loop: "{{ run__df_configs }}" + loop_control: + loop_var: __df_deploy_item + label: "{{ __df_deploy_item.name }}" + +# Wait for Service child deployments - name: Wait for CDP DE Virtual cluster setup to complete when: run__include_de + tags: de ansible.builtin.async_status: jid: "{{ __de_vc_build.ansible_job_id }}" register: __de_vc_builds_async @@ -218,3 +253,17 @@ loop_control: loop_var: __de_vc_build label: "{{ __de_vc_build.__de_vc_config_item.0.name | default ('None') }}" + +- name: Wait for CDP DF Deployments to complete + when: run__include_df + tags: df + ansible.builtin.async_status: + jid: "{{ __df_deployment.ansible_job_id }}" + register: __df_deployments_async + until: __df_deployments_async.finished + retries: 60 + delay: 15 + loop: "{{ __df_deployments.results }}" + loop_control: + loop_var: __df_deployment + label: "{{ __df_deployment.__df_deploy_item.0.name | default ('None') }}" \ No newline at end of file From 4bc3a5b64f71d2df3a04ed3e50bd819539cd9169 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Tue, 21 Dec 2021 18:11:09 +0000 Subject: [PATCH 3/5] Separate considerations for readyflows and customflows Add support for readyflows Remove comment support from df_service until string limitations are inline with rest of platform Add support for terminating deployed flows when disabling the DFX Service Signed-off-by: Daniel Chaffelson --- roles/common/defaults/main.yml | 8 ++++---- roles/runtime/defaults/main.yml | 2 ++ roles/runtime/tasks/setup_base.yml | 10 ++++++++++ roles/runtime/tasks/teardown_base.yml | 16 +++++++++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index ebcf64c6..bb8a327a 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -61,6 +61,7 @@ common__cdp_control_plane_region: "{{ globals.cdp_region | default('us-w common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" # Infra common__infra_deployment_engine: "{{ globals.infra_deployment_engine | default('ansible') }}" +common__aws_profile: "{{ globals.aws_profile | default('') }}" common__infra_type: "{{ globals.infra_type | default('aws') }}" common__public_key_file: "{{ globals.ssh.public_key_file | default('') }}" common__namespace_cdp: "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}" @@ -74,15 +75,15 @@ common__storage_name: "{{ infra.storage.name | default([comm common__terraform_base_dir: "{{ globals.terraform_base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}" # The processed Jinja template files for Terraform are placed in common__terraform_template_dir common__terraform_template_dir: "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}" -# A timestamped artefact directory storing a copy of the Terraform code from each run +# A timestamped artefact directory storing a copy of the Terraform code from each run common__terraform_artefact_dir: "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}" # Terraform apply/destroy run from under this directory common__terraform_workspace_dir: "{{ [common__terraform_base_dir, 'workspace'] | path_join }}" common__terraform_allowed_state_storage: "['local', 'remote_s3']" common__terraform_state_storage: "{{ globals.terraform_state_storage | default('local') }}" -common__terraform_remote_state_bucket: "{{ globals.terraform_remote_state_bucket | default('') }}" -common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}" +common__terraform_remote_state_bucket: "{{ globals.terraform_remote_state_bucket | default('') }}" +common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}" common__vpc_name: "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}" common__vpc_public_subnet_cidrs: "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}" @@ -109,7 +110,6 @@ common__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | de common__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" common__aws_region: "{{ infra.aws.region | default('eu-west-1') }}" -common__aws_profile: "{{ infra.aws.profile | default('') }}" common__aws_role_suffix: "{{ infra.aws.role.suffix | default(common__role_suffix) }}" common__aws_datalake_admin_role_name: "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}" common__aws_datalake_admin_suffix: "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 9734255f..ea8d968e 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -86,6 +86,8 @@ run__df_force_delete: "{{ df.force_delete | default(run__force_tea run__df_terminate_deployments: "{{ df.terminate_deployments | default(True) }}" run__df_tags: "{{ df.tags | default(common__tags) }}" run__df_deployments: "{{ df.deployments | default([]) }}" +run__df_readyflows: "{{ df.readyflows | default([]) }}" +run__df_delete_readyflows: "{{ df.delete_imported_readyflows | default(False) }}" # Deploy run__include_ml: "{{ common__include_ml }}" diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index 3c564147..f6babd8b 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -213,6 +213,16 @@ index_var: __de_vc_index label: "{{ __de_vc_config_item.0.name | default ('None') }}" +- name: Ensure requested CDP DF ReadyFlows are imported + when: run__include_df + tags: df + cloudera.cloud.df_readyflow: + name: "{{ __df_readyflow_item.flow_name }}" + loop: "{{ run__df_readyflows }}" + loop_control: + loop_var: __df_readyflow_item + label: "{{ __df_readyflow_item.flow_name }}" + - name: Create CDP DF Deployments when: run__include_df tags: df diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index ec5a028e..49deacef 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -190,7 +190,7 @@ retries: 120 delay: 30 -- name: Wait for CDP Dataflow deployment to decommission +- name: Wait for CDP Dataflow Service to decommission when: - run__include_df - run__df_service_info.services | length > 0 @@ -198,8 +198,22 @@ df_crn: "{{ __df_teardown_wait_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" + terminate: "{{ run__df_terminate_deployments }}" state: absent wait: yes loop_control: loop_var: __df_teardown_wait_item loop: "{{ run__df_service_info.services }}" + +- name: Ensure requested CDP DF ReadyFlow imports are deleted from Tenant + when: + - run__include_df + - run__df_delete_readyflows | bool + tags: df + cloudera.cloud.df_readyflow: + name: "{{ __df_readyflow_item.flow_name }}" + state: absent + loop: "{{ run__df_readyflows }}" + loop_control: + loop_var: __df_readyflow_item + label: "{{ __df_readyflow_item.flow_name }}" \ No newline at end of file From eb93ef8e26132ba6c357e1f85757d2bbeeb5e719 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Mon, 10 Jan 2022 14:21:45 +0000 Subject: [PATCH 4/5] DFX parameter groups submission set to arrays Added DFX to cloudera.exe.info role Set DFX Deployments to only be attempted when defined Added Example DFX Flow deployment to examples/cdf definition in cloudera-deploy Signed-off-by: Daniel Chaffelson --- roles/info/tasks/main.yml | 6 ++++++ roles/runtime/tasks/setup_base.yml | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/roles/info/tasks/main.yml b/roles/info/tasks/main.yml index 15a27953..3d023737 100644 --- a/roles/info/tasks/main.yml +++ b/roles/info/tasks/main.yml @@ -80,6 +80,11 @@ env: "{{ info__env_name }}" register: __opdb_info +- name: Query CDP DFX + cloudera.cloud.df_service_info: + name: "{{ info__env_name }}" + register: __df_info + - name: Set facts for the CDP deployment details ansible.builtin.set_fact: deployment: @@ -87,6 +92,7 @@ datalake: "{{ __datalake_info.datalakes | first | default({}) }}" datahubs: "{{ __datahubs_info.datahubs }}" workspaces: "{{ __ml_info.workspaces }}" + dataflow: "{{ __df_info.services }}" services: "{{ __de_info.services }}" operational_dbs: "{{ __opdb_info.databases }}" diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index f6babd8b..8d150678 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -224,7 +224,9 @@ label: "{{ __df_readyflow_item.flow_name }}" - name: Create CDP DF Deployments - when: run__include_df + when: + - run__include_df + - run__df_configs is defined tags: df cloudera.cloud.df_deployment: name: "{{ __df_deploy_item.name }}" @@ -265,7 +267,9 @@ label: "{{ __de_vc_build.__de_vc_config_item.0.name | default ('None') }}" - name: Wait for CDP DF Deployments to complete - when: run__include_df + when: + - run__include_df + - run__df_configs is defined tags: df ansible.builtin.async_status: jid: "{{ __df_deployment.ansible_job_id }}" From 98cf47f5c112a3cff50bc23dc6a82266868236a7 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 24 Mar 2022 20:03:31 +0000 Subject: [PATCH 5/5] CDF Updates per reviews Improved application deployment playbook to only attempt Kafka flow deployment when Kafka Datahub is found Improved cdpy CRN validation by moving substring definitions to a constant and simplifying the validation logic Improved cdpy.df.describe_service to only run if the dataflow CRN is correctly resolved Force renamed readyflowCRN for an imported ReadyFlow to be addedReadyflowCrn to improve usability Renamed some tasks to improve distinction between similar but different activities like enabling the DF service vs deploying a DF flow Corrected various minor documentation points for DF modules Normalized responses to readyflow_info, deployment_info, and customflow_info to use listings of the full description of objects to simplify user experience Signed-off-by: Daniel Chaffelson --- roles/runtime/tasks/setup_base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index 8d150678..481d2c92 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -180,7 +180,7 @@ retries: 120 delay: 30 -- name: Wait for CDP Dataflow deployment to complete +- name: Wait for CDP Dataflow Service Enablement to complete when: run__include_df tags: df cloudera.cloud.df_service: