diff --git a/docs/configuration.yml b/docs/configuration.yml index 2400601f..5ce97a6d 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -33,6 +33,37 @@ df: persist: force_delete: terminate_deployments: + tags: + deployments: + - name: + flow_ver_crn: + size: + static_node_count: + autoscale: + autoscale_nodes_min: + autoscale_nodes_max: + nifi_ver: + autostart_flow: + parameter_groups: + - name: + parameters: + - name: + value: + assetReferences: [str, ...] + kpis: + - metricId: + componentId: + alert: + thresholdMoreThan: + unitId: + value: + thresholdLessThan: + unitId: + value: + frequencyTolerance: + value: + unit: + id: SECONDS|MINUTES|HOURS|DAYS dw: definitions: suffix: diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index ebcf64c6..bb8a327a 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -61,6 +61,7 @@ common__cdp_control_plane_region: "{{ globals.cdp_region | default('us-w common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" # Infra common__infra_deployment_engine: "{{ globals.infra_deployment_engine | default('ansible') }}" +common__aws_profile: "{{ globals.aws_profile | default('') }}" common__infra_type: "{{ globals.infra_type | default('aws') }}" common__public_key_file: "{{ globals.ssh.public_key_file | default('') }}" common__namespace_cdp: "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}" @@ -74,15 +75,15 @@ common__storage_name: "{{ infra.storage.name | default([comm common__terraform_base_dir: "{{ globals.terraform_base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}" # The processed Jinja template files for Terraform are placed in common__terraform_template_dir common__terraform_template_dir: "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}" -# A timestamped artefact directory storing a copy of the Terraform code from each run +# A timestamped artefact directory storing a copy of the Terraform code from each run common__terraform_artefact_dir: "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}" # Terraform apply/destroy run from under this directory common__terraform_workspace_dir: "{{ [common__terraform_base_dir, 'workspace'] | path_join }}" common__terraform_allowed_state_storage: "['local', 'remote_s3']" common__terraform_state_storage: "{{ globals.terraform_state_storage | default('local') }}" -common__terraform_remote_state_bucket: "{{ globals.terraform_remote_state_bucket | default('') }}" -common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}" +common__terraform_remote_state_bucket: "{{ globals.terraform_remote_state_bucket | default('') }}" +common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}" common__vpc_name: "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}" common__vpc_public_subnet_cidrs: "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}" @@ -109,7 +110,6 @@ common__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | de common__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" common__aws_region: "{{ infra.aws.region | default('eu-west-1') }}" -common__aws_profile: "{{ infra.aws.profile | default('') }}" common__aws_role_suffix: "{{ infra.aws.role.suffix | default(common__role_suffix) }}" common__aws_datalake_admin_role_name: "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}" common__aws_datalake_admin_suffix: "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" diff --git a/roles/info/tasks/main.yml b/roles/info/tasks/main.yml index 15a27953..3d023737 100644 --- a/roles/info/tasks/main.yml +++ b/roles/info/tasks/main.yml @@ -80,6 +80,11 @@ env: "{{ info__env_name }}" register: __opdb_info +- name: Query CDP DFX + cloudera.cloud.df_service_info: + name: "{{ info__env_name }}" + register: __df_info + - name: Set facts for the CDP deployment details ansible.builtin.set_fact: deployment: @@ -87,6 +92,7 @@ datalake: "{{ __datalake_info.datalakes | first | default({}) }}" datahubs: "{{ __datahubs_info.datahubs }}" workspaces: "{{ __ml_info.workspaces }}" + dataflow: "{{ __df_info.services }}" services: "{{ __de_info.services }}" operational_dbs: "{{ __opdb_info.databases }}" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 765152fe..ea8d968e 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -83,7 +83,11 @@ run__df_cluster_subnets: "{{ df.cluster_subnets | default([]) }}" run__df_lb_subnets: "{{ df.loadbalancer_subnets | default([]) }}" run__df_persist: "{{ df.teardown.persist | default(False) }}" run__df_force_delete: "{{ df.force_delete | default(run__force_teardown) }}" -run__df_terminate_deployments: "{{ df.terminate_deployments | default(run__force_teardown) }}" +run__df_terminate_deployments: "{{ df.terminate_deployments | default(True) }}" +run__df_tags: "{{ df.tags | default(common__tags) }}" +run__df_deployments: "{{ df.deployments | default([]) }}" +run__df_readyflows: "{{ df.readyflows | default([]) }}" +run__df_delete_readyflows: "{{ df.delete_imported_readyflows | default(False) }}" # Deploy run__include_ml: "{{ common__include_ml }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index b2d84d4f..06ccbc11 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -166,6 +166,7 @@ label: "{{ config.name }}" - name: Prepare for CDP DE Service experiences + tags: de when: run__include_de block: - name: Construct CDP DE Service configurations @@ -181,3 +182,22 @@ loop_control: loop_var: __de_config label: "{{ config.name }}" + +- name: Prepare for CDP DF Service experiences + tags: df + when: + - run__include_df + - run__df_deployments | length > 0 + block: + - name: Construct CDP DF Deployment configurations + ansible.builtin.set_fact: + run__df_configs: "{{ run__df_configs | default([]) | union([config]) }}" + vars: + include: "{{ lookup('template', __df_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" + config: + name: "{{ __df_config.name | default([run__namespace_cdp, __df_config.flow_name[::2] | replace(' ','') ] | join('-')) }}" + raw: "{{ __df_config }}" + loop: "{{ run__df_deployments }}" + loop_control: + loop_var: __df_config + label: "{{ config.name }}" diff --git a/roles/runtime/tasks/initialize_setup.yml b/roles/runtime/tasks/initialize_setup.yml index 40bc06a6..25d20c4c 100644 --- a/roles/runtime/tasks/initialize_setup.yml +++ b/roles/runtime/tasks/initialize_setup.yml @@ -29,3 +29,5 @@ - dw - opdb - dh + - df + - de diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index 2baaba83..5ec802a1 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -27,7 +27,7 @@ - name: Discover CDP DF Deployments register: run__df_service_info when: run__include_df - cloudera.cloud.df_info: + cloudera.cloud.df_service_info: name: "{{ run__env_name }}" - name: Initialize Purge of all Runtimes in Environment diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index a1991f6d..481d2c92 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Request Service Deployments - name: Request CDP Datahub deployments when: run__include_datahub tags: dh @@ -63,6 +64,7 @@ - name: Execute CDP DE Service experiences setup when: run__include_de + tags: de cloudera.cloud.de: name: "{{ __de_config_item.name }}" env: "{{ run__env_name }}" @@ -117,9 +119,11 @@ kube_ip_ranges: "{{ run__df_kube_ip_ranges }}" cluster_subnets: "{{ run__df_cluster_subnets }}" loadbalancer_subnets: "{{ run__df_lb_subnets }}" + tags: "{{ run__df_tags }}" state: present wait: no +# Wait for Service Deployments - name: Wait for CDP Datahub deployments to complete when: run__include_datahub tags: dh @@ -150,6 +154,7 @@ - name: Wait for CDP DE Service experiences to complete when: run__include_de + tags: de ansible.builtin.async_status: jid: "{{ __de_build.ansible_job_id }}" loop_control: @@ -175,15 +180,17 @@ retries: 120 delay: 30 -- name: Wait for CDP Dataflow deployment to complete +- name: Wait for CDP Dataflow Service Enablement to complete when: run__include_df tags: df cloudera.cloud.df_service: env_crn: "{{ run__cdp_env_crn }}" wait: yes +# Request Service child deployments - name: Create CDP DE Virtual clusters when: run__include_de + tags: de cloudera.cloud.de_virtual_cluster: cluster_name: "{{ __de_vc_config_item.0.name }}" env: "{{ run__env_name }}" @@ -206,8 +213,48 @@ index_var: __de_vc_index label: "{{ __de_vc_config_item.0.name | default ('None') }}" +- name: Ensure requested CDP DF ReadyFlows are imported + when: run__include_df + tags: df + cloudera.cloud.df_readyflow: + name: "{{ __df_readyflow_item.flow_name }}" + loop: "{{ run__df_readyflows }}" + loop_control: + loop_var: __df_readyflow_item + label: "{{ __df_readyflow_item.flow_name }}" + +- name: Create CDP DF Deployments + when: + - run__include_df + - run__df_configs is defined + tags: df + cloudera.cloud.df_deployment: + name: "{{ __df_deploy_item.name }}" + df_name: "{{ run__env_name }}" + flow_ver_crn: "{{ __df_deploy_item.raw.flow_ver_crn | default(omit) }}" + flow_name: "{{ __df_deploy_item.raw.flow_name | default(omit) }}" + flow_ver: "{{ __df_deploy_item.raw.flow_ver | default(omit) }}" + size: "{{ __df_deploy_item.raw.size | default(omit) }}" + static_node_count: "{{ __df_deploy_item.raw.static_node_count | default(omit) }}" + autoscale: "{{ __df_deploy_item.raw.autoscale | default(omit) }}" + autoscale_nodes_min: "{{ __df_deploy_item.raw.autoscale_nodes_min | default(omit) }}" + autoscale_nodes_max: "{{ __df_deploy_item.raw.autoscale_nodes_max | default(omit) }}" + nifi_ver: "{{ __df_deploy_item.raw.nifi_ver | default(omit) }}" + autostart_flow: "{{ __df_deploy_item.raw.autostart_flow | default(omit) }}" + parameter_groups: "{{ __df_deploy_item.raw.parameter_groups | default(omit) }}" + kpis: "{{ __df_deploy_item.raw.kpis | default(omit) }}" + async: 720 + poll: 0 + register: __df_deployments + loop: "{{ run__df_configs }}" + loop_control: + loop_var: __df_deploy_item + label: "{{ __df_deploy_item.name }}" + +# Wait for Service child deployments - name: Wait for CDP DE Virtual cluster setup to complete when: run__include_de + tags: de ansible.builtin.async_status: jid: "{{ __de_vc_build.ansible_job_id }}" register: __de_vc_builds_async @@ -218,3 +265,19 @@ loop_control: loop_var: __de_vc_build label: "{{ __de_vc_build.__de_vc_config_item.0.name | default ('None') }}" + +- name: Wait for CDP DF Deployments to complete + when: + - run__include_df + - run__df_configs is defined + tags: df + ansible.builtin.async_status: + jid: "{{ __df_deployment.ansible_job_id }}" + register: __df_deployments_async + until: __df_deployments_async.finished + retries: 60 + delay: 15 + loop: "{{ __df_deployments.results }}" + loop_control: + loop_var: __df_deployment + label: "{{ __df_deployment.__df_deploy_item.0.name | default ('None') }}" \ No newline at end of file diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index ec5a028e..49deacef 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -190,7 +190,7 @@ retries: 120 delay: 30 -- name: Wait for CDP Dataflow deployment to decommission +- name: Wait for CDP Dataflow Service to decommission when: - run__include_df - run__df_service_info.services | length > 0 @@ -198,8 +198,22 @@ df_crn: "{{ __df_teardown_wait_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" + terminate: "{{ run__df_terminate_deployments }}" state: absent wait: yes loop_control: loop_var: __df_teardown_wait_item loop: "{{ run__df_service_info.services }}" + +- name: Ensure requested CDP DF ReadyFlow imports are deleted from Tenant + when: + - run__include_df + - run__df_delete_readyflows | bool + tags: df + cloudera.cloud.df_readyflow: + name: "{{ __df_readyflow_item.flow_name }}" + state: absent + loop: "{{ run__df_readyflows }}" + loop_control: + loop_var: __df_readyflow_item + label: "{{ __df_readyflow_item.flow_name }}" \ No newline at end of file