From 5c71df7993ca41274aa634ddd7fad0f61637ae92 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Wed, 15 Sep 2021 10:49:48 +0100 Subject: [PATCH 1/3] Move DFX Beta implementation to GA process Cdpy: * Rework cdpy/df to use the 'service' nomenclature instead of 'environment' for DataFlow * Allow separate submission of a name, env_crn, or df_crn for filtering the list of enabled DataFlow services * Allow submission of an env_crn or df_crn for the description of a Dataflow Service, as you cannot know the df_crn during creation until it is actually created * Standardise on use of df_crn for the service crn in DataFlow to provide differentiation from the env crn and other crns in CDP * Add new service enablement parameters to the dfx enable_service call * Add new terminate option to the DFX disable_service call * Change delete_environment to reset_service to reflect new usage * Squelch 403 response on describing a recently deleted Dataflow Service while it is being cleaned up cloudera.cloud: * renamed df.py to df_service.py to reflect that other DataFlow modules for Deployments will be needed in future * Include changes to submission parameters and method names for 'cdp df' calls in cdpy * Modify enable and disable service logic to match new methods and parameters * df_info now supports name, df_crn, or env_crn for filtering results * env_info once again checks for DataFlow during detailed Environment information collection * Added input validation for DataFlow service creation cloudera.exe: * Added defaults handling for new DataFlow submission parameters * renamed various methods and parameters to reflect updated DataFlow service naming and options Signed-off-by: Daniel Chaffelson --- roles/runtime/defaults/main.yml | 8 ++++++-- roles/runtime/tasks/initialize_teardown.yml | 2 +- roles/runtime/tasks/setup_base.yml | 13 ++++++++----- roles/runtime/tasks/teardown_base.yml | 19 ++++++++++--------- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 8cce89e2..f685b4ac 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -71,9 +71,13 @@ run__dw_force_delete: "{{ dw.force_delete | default (run__force_te run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" run__df_public_loadbalancer: "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}" -run__df_ip_ranges: "{{ df.ip_ranges | default([]) }}" +run__df_lb_ip_ranges: "{{ df.loadbalancer_ip_ranges | default([]) }}" +run__df_kube_ip_ranges: "{{ df.kube_ip_ranges | default([]) }}" +run__df_cluster_subnets: "{{ df.cluster_subnets | default([]) }}" +run__df_lb_subnets: "{{ df.loadbalancer_subnets | default([]) }}" run__df_persist: "{{ df.teardown.persist | default(False) }}" -run__df_force_delete: "{{ df.force_delete | default (run__force_teardown) }}" +run__df_force_delete: "{{ df.force_delete | default(run__force_teardown) }}" +run__df_terminate_deployments: "{{ df.terminate_deployments | default(run__force_teardown) }}" # Deploy run__include_ml: "{{ common__include_ml }}" diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index b798bfa8..553e4ba1 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -25,7 +25,7 @@ ansible.builtin.include_tasks: "initialize_base.yml" - name: Discover CDP DF Deployments - register: run__df_env_info + register: run__df_service_info when: run__include_df cloudera.cloud.df_info: name: "{{ run__env_name }}" diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index 6aba689a..5d34c6b9 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -76,12 +76,15 @@ - name: Execute CDP Dataflow setup when: run__include_df - cloudera.cloud.df: - name: "{{ run__cdp_env_crn }}" + cloudera.cloud.df_service: + env_crn: "{{ run__cdp_env_crn }}" nodes_min: "{{ run__df_nodes_min }}" nodes_max: "{{ run__df_nodes_max }}" public_loadbalancer: "{{ run__df_public_loadbalancer }}" - ip_ranges: "{{ run__df_ip_ranges }}" + loadbalancer_ip_ranges: "{{ run__df_lb_ip_ranges }}" + kube_ip_ranges: "{{ run__df_kube_ip_ranges }}" + cluster_subnets: "{{ run__df_cluster_subnets }}" + loadbalancer_subnets: "{{ run__df_lb_subnets }}" state: present wait: no @@ -126,6 +129,6 @@ - name: Wait for CDP Dataflow deployment to complete when: run__include_df - cloudera.cloud.df: - name: "{{ run__cdp_env_crn }}" + cloudera.cloud.df_service: + env_crn: "{{ run__cdp_env_crn }}" wait: yes diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index a5707698..4464dfe3 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -48,17 +48,18 @@ register: __df_teardown_info when: - run__include_df or run__force_teardown | bool - - run__df_env_info is defined and run__df_env_info.services is defined - - run__df_env_info.services | length > 0 - cloudera.cloud.df: - name: "{{ __df_teardown_req_item.crn }}" + - run__df_service_info is defined and run__df_service_info.services is defined + - run__df_service_info.services | length > 0 + cloudera.cloud.df_service: + df_crn: "{{ __df_teardown_req_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" + terminate: "{{ run__df_terminate_deployments }}" state: absent wait: no loop_control: loop_var: __df_teardown_req_item - loop: "{{ run__df_env_info.services }}" + loop: "{{ run__df_service_info.services }}" - name: Execute CDP ML Workspace teardown when: @@ -158,13 +159,13 @@ - name: Wait for CDP Dataflow deployment to decommission when: - run__include_df - - run__df_env_info.services | length > 0 - cloudera.cloud.df: - name: "{{ __df_teardown_wait_item.crn }}" + - run__df_service_info.services | length > 0 + cloudera.cloud.df_service: + df_crn: "{{ __df_teardown_wait_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" state: absent wait: yes loop_control: loop_var: __df_teardown_wait_item - loop: "{{ run__df_env_info.services }}" \ No newline at end of file + loop: "{{ run__df_service_info.services }}" \ No newline at end of file From df4d4f63e9f7d50869b2034dca240fec76233fa1 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Wed, 15 Sep 2021 14:56:13 +0100 Subject: [PATCH 2/3] Add Dataflow roles to the default roles provisioned to the default CDP user groups Update the cloudera-deploy CDF example definition to have the GA parameters by default Signed-off-by: Daniel Chaffelson --- roles/platform/vars/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/platform/vars/main.yml b/roles/platform/vars/main.yml index fd362b8a..f1109a96 100644 --- a/roles/platform/vars/main.yml +++ b/roles/platform/vars/main.yml @@ -39,6 +39,9 @@ plat__cdp_iam_identities: dw_user: DWUser ml_admin: MLAdmin ml_user: MLUser + df_admin: DFAdmin + df_flow_admin: DFFlowAdmin + df_flow_user: DFFlowUser plat__cdp_iam_admin_group_resource_roles_default: - "{{ plat__cdp_iam_identities.env_admin }}" @@ -47,11 +50,16 @@ plat__cdp_iam_admin_group_resource_roles_default: - "{{ plat__cdp_iam_identities.dw_user }}" - "{{ plat__cdp_iam_identities.ml_admin }}" - "{{ plat__cdp_iam_identities.ml_user }}" + - "{{ plat__cdp_iam_identities.df_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_user }}" plat__cdp_iam_user_group_resource_roles_default: - "{{ plat__cdp_iam_identities.env_user }}" - "{{ plat__cdp_iam_identities.dw_user }}" - "{{ plat__cdp_iam_identities.ml_user }}" + - "{{ plat__cdp_iam_identities.df_flow_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_user }}" plat__gcp_xaccount_policy_bindings_default: - "roles/compute.instanceAdmin.v1" From 08e1407cb1e0ddcd3e7b5b83c02bf74a329bc19e Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 16 Sep 2021 16:47:42 +0100 Subject: [PATCH 3/3] DFX may set deletion protection for RDS Instances, therefore it must be disabled if purging the infrastructure Signed-off-by: Daniel Chaffelson --- .../tasks/teardown_aws_compute.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/roles/infrastructure/tasks/teardown_aws_compute.yml b/roles/infrastructure/tasks/teardown_aws_compute.yml index 1e21f2cd..bc269dd0 100644 --- a/roles/infrastructure/tasks/teardown_aws_compute.yml +++ b/roles/infrastructure/tasks/teardown_aws_compute.yml @@ -42,6 +42,19 @@ poll: 0 register: __eks_teardowns_info +- name: Ensure RDS Instance deletion protection is disabled if discovered + when: + - infra__force_teardown | bool + - __infra_aws_rds_instances is defined + - __infra_aws_rds_instances | length > 0 + command: > + aws rds modify-db-instance + --db-instance-identifier {{ __infra_rds_protect_item.db_instance_identifier }} + --no-deletion-protection + loop: "{{ __infra_aws_rds_instances }}" + loop_control: + loop_var: __infra_rds_protect_item + - name: Handle RDS Instance removal if discovered when: - infra__force_teardown | bool @@ -55,6 +68,7 @@ loop: "{{ __infra_aws_rds_instances }}" loop_control: loop_var: __infra_rds_remove_item + label: "{{ __infra_rds_remove_item.db_instance_identifier }}" async: 3600 # 1 hour timeout poll: 0 register: __rds_teardowns_info @@ -71,7 +85,7 @@ loop: "{{ __rds_teardowns_info.results }}" register: __rds_teardowns_async until: __rds_teardowns_async.finished - retries: 360 + retries: 3600 delay: 10 - name: Wait for EKS teardowns to complete