From d6272d376152a448490bdbd53a8336be41fc4a97 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:24 -0500 Subject: [PATCH 01/34] Support CDW VW changes Signed-off-by: Saravanan Raju --- roles/common/defaults/main.yml | 2 +- roles/runtime/defaults/main.yml | 7 +- roles/runtime/tasks/initialize_base.yml | 48 ++++++++--- roles/runtime/tasks/setup_aws.yml | 109 ++++++++++++++++++++---- 4 files changed, 137 insertions(+), 29 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 81b0a69f..217dd889 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -128,7 +128,7 @@ common__env_name_suffix: "{{ env.suffix | default(common__env_s common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" -common__tunnel: "{{ env.tunnel | default(False) }}" +common__tunnel: "{{ env.tunnel | default(True) }}" common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" common__env_admin_password: "{{ globals.admin_password | mandatory }}" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 765152fe..33d7a859 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -71,7 +71,12 @@ run__de_force_delete: "{{ de.force_delete | default (run__force_te run__de_vc_suffix: "{{ de.vc.suffix | default('vc') }}" run__dw_definitions: "{{ dw.definitions | default([{}]) }}" -run__dw_suffix: "{{ dw.suffix | default('dw') }}" +run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" +run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" +run__dw_tags: "{{ dw.tags | default(common__tags) }}" +run__dw_overlay_network: "{{ dw.overlay_network | bool }}" +run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index b2d84d4f..c9ced135 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -165,19 +165,45 @@ loop_var: __ml_config label: "{{ config.name }}" -- name: Prepare for CDP DE Service experiences - when: run__include_de +- name: Prepare for CDP DW experiences + when: run__include_dw block: - - name: Construct CDP DE Service configurations + - name: Construct CDP DW Data catalog configurations ansible.builtin.set_fact: - run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}" + run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: - include: "{{ lookup('template', __de_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - overlay_tags: "{{ __de_config.tags | default(include.tags) | default(run__de_tags) }}" + include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" + use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}" + default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}" config: - name: "{{ __de_config.name | default([run__namespace_cdp, __de_config.suffix | default(include.suffix) | default(run__de_suffix)] | join('-')) }}" - raw: "{{ __de_config }}" - loop: "{{ run__de_definitions }}" + name: "{{ __dw_config.name | default(default_dbc_name) }}" + load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" + use_default_dbc: "{{ use_default_dbc }}" + virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" + loop: "{{ run__dw_definitions }}" loop_control: - loop_var: __de_config - label: "{{ config.name }}" + loop_var: __dw_config + index_var: __dw_config_index + + - name: Construct CDP DW Virtual warehouse configurations + ansible.builtin.set_fact: + run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" + vars: + dbc_name: "{{ item.0.name }}" + use_default_dbc: "{{ item.0.use_default_dbc }}" + config: + name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + dbc_name: "{{ dbc_name }}" + use_default_dbc: "{{ use_default_dbc }}" + vw_type: "{{ item.1.vw_type | default('hive') }}" + template: "{{ item.1.template | default('xsmall') }}" + autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}" + autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}" + common_configs: "{{ item.1.configs.common_configs | default({}) }}" + application_configs: "{{ item.1.configs.application_configs | default({}) }}" + ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" + enable_sso: "{{ item.1.configs.enable_sso | default(None) }}" + tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}" + loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" + loop_control: + index_var: __dw_dbc_index diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index b53c18ae..b4201be7 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -39,27 +39,104 @@ when: run__include_dw tags: dw block: - - name: Execute CDP DW cluster setup - cloudera.cloud.dw_cluster: +# - name: Execute CDP DW cluster setup +# cloudera.cloud.dw_cluster: +# env: "{{ run__env_name }}" +# overlay: "{{ run__dw_overlay_network }}" +# aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" +# aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" +# state: present +# wait: yes +# async: 3600 # 1 hour timeout +# poll: 0 +# register: __dw_builds +# +# - name: Wait for CDP DW cluster setup to complete +# ansible.builtin.async_status: +# jid: "{{ __dw_builds.ansible_job_id }}" +# #loop_control: +# # loop_var: __opdb_build +# # label: "{{ __opdb_build.__opdb_config.name }}" +# #loop: "{{ __opdb_builds.results }}" +# register: __dw_builds_async +# until: __dw_builds_async.finished +# retries: 120 +# delay: 30 + + - name: Retrieve CDP DW experiences + cloudera.cloud.dw_cluster_info: env: "{{ run__env_name }}" - overlay: no - # TODO - Allow direct assignment (will need to coordinate with infra role) - aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + register: run__dw_list + + - name: Create CDP DW Database catalogs + when: not __dw_dbc_config.use_default_dbc + cloudera.cloud.dw_dbc: + cluster_id : "{{ run__dw_list.clusters[0].id }}" + name: "{{ __dw_dbc_config.name }}" + load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" state: present wait: yes async: 3600 # 1 hour timeout poll: 0 - register: __dw_builds + loop: "{{ run__dw_dbc_configs }}" + loop_control: + loop_var: __dw_dbc_config + register: __dw_dbc_builds - - name: Wait for CDP DW cluster setup to complete + - name: Wait for CDP DW Database catalogs setup to complete + when: __dw_dbc_build.ansible_job_id is defined ansible.builtin.async_status: - jid: "{{ __dw_builds.ansible_job_id }}" - #loop_control: - # loop_var: __opdb_build - # label: "{{ __opdb_build.__opdb_config.name }}" - #loop: "{{ __opdb_builds.results }}" - register: __dw_builds_async - until: __dw_builds_async.finished + jid: "{{ __dw_dbc_build.ansible_job_id }}" + register: __dw_dbc_builds_async + until: __dw_dbc_builds_async.finished retries: 120 - delay: 30 \ No newline at end of file + delay: 30 + loop: "{{ __dw_dbc_builds.results }}" + loop_control: + loop_var: __dw_dbc_build + + - name: Set CDP DW Database catalog name to id map + when: __dw_dbc_build_async.dbcs is defined + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}" + loop: "{{ __dw_dbc_builds_async.results }}" + loop_control: + loop_var: __dw_dbc_build_async + + - name: Create CDP DW Virtual warehouse + cloudera.cloud.dw_vw: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" + vw_type: "{{ __dw_vw_config.vw_type }}" + name: "{{ __dw_vw_config.name }}" + template: "{{ __dw_vw_config.template }}" + autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}" + autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}" + common_configs: "{{ __dw_vw_config.common_configs }}" + application_configs: "{{ __dw_vw_config.application_configs }}" + ldap_groups: "{{ __dw_vw_config.ldap_groups }}" + enable_sso: "{{ __dw_vw_config.enable_sso | bool }}" + tags: "{{ __dw_vw_config.tags }}" + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_vw_builds + loop: "{{ run__dw_vw_configs }}" + loop_control: + loop_var: __dw_vw_config + + - name: Wait for CDP DW Virtual warehouse setup to complete + when: __dw_vw_build.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __dw_vw_build.ansible_job_id }}" + register: __dw_vw_builds_async + until: __dw_vw_builds_async.finished + retries: 120 + delay: 30 + loop: "{{ __dw_vw_builds.results }}" + loop_control: + loop_var: __dw_vw_build + + - name: Print Virtual Warehouses + ansible.builtin.debug: + msg: "Database Cluster : {{ __dw_vw_builds_async }}" \ No newline at end of file From 682586beb69050b37450f268841fba8d7089ee68 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:25 -0500 Subject: [PATCH 02/34] Revert unnecessary changes Signed-off-by: Saravanan Raju --- roles/common/defaults/main.yml | 2 +- roles/runtime/tasks/setup_aws.yml | 46 +++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 217dd889..81b0a69f 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -128,7 +128,7 @@ common__env_name_suffix: "{{ env.suffix | default(common__env_s common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" -common__tunnel: "{{ env.tunnel | default(True) }}" +common__tunnel: "{{ env.tunnel | default(False) }}" common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" common__env_admin_password: "{{ globals.admin_password | mandatory }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index b4201be7..2c332822 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -39,29 +39,29 @@ when: run__include_dw tags: dw block: -# - name: Execute CDP DW cluster setup -# cloudera.cloud.dw_cluster: -# env: "{{ run__env_name }}" -# overlay: "{{ run__dw_overlay_network }}" -# aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" -# aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" -# state: present -# wait: yes -# async: 3600 # 1 hour timeout -# poll: 0 -# register: __dw_builds -# -# - name: Wait for CDP DW cluster setup to complete -# ansible.builtin.async_status: -# jid: "{{ __dw_builds.ansible_job_id }}" -# #loop_control: -# # loop_var: __opdb_build -# # label: "{{ __opdb_build.__opdb_config.name }}" -# #loop: "{{ __opdb_builds.results }}" -# register: __dw_builds_async -# until: __dw_builds_async.finished -# retries: 120 -# delay: 30 + - name: Execute CDP DW cluster setup + cloudera.cloud.dw_cluster: + env: "{{ run__env_name }}" + overlay: "{{ run__dw_overlay_network }}" + aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" + aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + state: present + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_builds + + - name: Wait for CDP DW cluster setup to complete + ansible.builtin.async_status: + jid: "{{ __dw_builds.ansible_job_id }}" + #loop_control: + # loop_var: __opdb_build + # label: "{{ __opdb_build.__opdb_config.name }}" + #loop: "{{ __opdb_builds.results }}" + register: __dw_builds_async + until: __dw_builds_async.finished + retries: 120 + delay: 30 - name: Retrieve CDP DW experiences cloudera.cloud.dw_cluster_info: From 62da5b80031e65566713ccc06f87df029071653d Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:25 -0500 Subject: [PATCH 03/34] Add labels for the loops Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 3 ++- roles/runtime/tasks/setup_aws.yml | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index c9ced135..6695b836 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -192,7 +192,7 @@ dbc_name: "{{ item.0.name }}" use_default_dbc: "{{ item.0.use_default_dbc }}" config: - name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" dbc_name: "{{ dbc_name }}" use_default_dbc: "{{ use_default_dbc }}" vw_type: "{{ item.1.vw_type | default('hive') }}" @@ -207,3 +207,4 @@ loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: index_var: __dw_dbc_index + label: "{{ item.0.name }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 2c332822..2f22fdb4 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -43,8 +43,9 @@ cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" overlay: "{{ run__dw_overlay_network }}" + private_load_balancer: "{{ run__dw_private_load_balancer }}" aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present wait: yes async: 3600 # 1 hour timeout @@ -81,6 +82,7 @@ loop: "{{ run__dw_dbc_configs }}" loop_control: loop_var: __dw_dbc_config + label: "{{ __dw_dbc_config.name }}" register: __dw_dbc_builds - name: Wait for CDP DW Database catalogs setup to complete @@ -94,6 +96,7 @@ loop: "{{ __dw_dbc_builds.results }}" loop_control: loop_var: __dw_dbc_build + label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map when: __dw_dbc_build_async.dbcs is defined @@ -124,6 +127,7 @@ loop: "{{ run__dw_vw_configs }}" loop_control: loop_var: __dw_vw_config + label: "{{ __dw_vw_config.name }}" - name: Wait for CDP DW Virtual warehouse setup to complete when: __dw_vw_build.ansible_job_id is defined @@ -136,7 +140,4 @@ loop: "{{ __dw_vw_builds.results }}" loop_control: loop_var: __dw_vw_build - - - name: Print Virtual Warehouses - ansible.builtin.debug: - msg: "Database Cluster : {{ __dw_vw_builds_async }}" \ No newline at end of file + label: "{{ __dw_vw_build.__dw_vw_config.name }}" \ No newline at end of file From 90c8ee7b997c9061ea94a7c2a73e649f55e4d18d Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:25 -0500 Subject: [PATCH 04/34] Remove use_default_dbc flag Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 6695b836..e2e3a3a4 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -173,10 +173,9 @@ run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}" - default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}" + use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}" config: - name: "{{ __dw_config.name | default(default_dbc_name) }}" + name: "{{ __dw_config.name | default('') }}" load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" use_default_dbc: "{{ use_default_dbc }}" virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" From a00d17e8ed283e17c71e065c47e1a5f4197ef9a6 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:25 -0500 Subject: [PATCH 05/34] Change the vw and dbc module names and other minor refactor Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 6 +++--- roles/runtime/tasks/setup_aws.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index e2e3a3a4..8dd0f9c1 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -194,10 +194,10 @@ name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" dbc_name: "{{ dbc_name }}" use_default_dbc: "{{ use_default_dbc }}" - vw_type: "{{ item.1.vw_type | default('hive') }}" + type: "{{ item.1.type | default('hive') }}" template: "{{ item.1.template | default('xsmall') }}" - autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}" - autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}" + autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" + autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}" common_configs: "{{ item.1.configs.common_configs | default({}) }}" application_configs: "{{ item.1.configs.application_configs | default({}) }}" ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 2f22fdb4..b32fd168 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -71,7 +71,7 @@ - name: Create CDP DW Database catalogs when: not __dw_dbc_config.use_default_dbc - cloudera.cloud.dw_dbc: + cloudera.cloud.dw_database_catalog: cluster_id : "{{ run__dw_list.clusters[0].id }}" name: "{{ __dw_dbc_config.name }}" load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" @@ -99,22 +99,22 @@ label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map - when: __dw_dbc_build_async.dbcs is defined + when: __dw_dbc_build_async.database_catalogs is defined ansible.builtin.set_fact: - run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}" + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async - name: Create CDP DW Virtual warehouse - cloudera.cloud.dw_vw: + cloudera.cloud.dw_virtual_warehouse: cluster_id: "{{ run__dw_list.clusters[0].id }}" dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" - vw_type: "{{ __dw_vw_config.vw_type }}" + type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}" - autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}" common_configs: "{{ __dw_vw_config.common_configs }}" application_configs: "{{ __dw_vw_config.application_configs }}" ldap_groups: "{{ __dw_vw_config.ldap_groups }}" From 18ad18ed78d5c886f60b62fb91ddbf7596f2ba29 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:25 -0500 Subject: [PATCH 06/34] Minor refactor Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 8dd0f9c1..3f28e8db 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -188,12 +188,10 @@ ansible.builtin.set_fact: run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: - dbc_name: "{{ item.0.name }}" - use_default_dbc: "{{ item.0.use_default_dbc }}" config: name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" - dbc_name: "{{ dbc_name }}" - use_default_dbc: "{{ use_default_dbc }}" + dbc_name: "{{ item.0.name }}" + use_default_dbc: "{{ item.0.use_default_dbc }}" type: "{{ item.1.type | default('hive') }}" template: "{{ item.1.template | default('xsmall') }}" autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" @@ -206,4 +204,4 @@ loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: index_var: __dw_dbc_index - label: "{{ item.0.name }}" + label: "{{ item.0.name }}" \ No newline at end of file From ff9b22fbff64002d03d35b1b6d82a2e42a900033 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Wed, 24 Nov 2021 12:13:26 -0500 Subject: [PATCH 07/34] Address review comments Signed-off-by: Saravanan Raju --- roles/runtime/defaults/main.yml | 3 +++ roles/runtime/tasks/initialize_base.yml | 30 ++++++++++++------------- roles/runtime/tasks/setup_aws.yml | 23 ++++++++++++++----- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 33d7a859..2ef56cc5 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -78,6 +78,9 @@ run__dw_overlay_network: "{{ dw.overlay_network | bool }}" run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" +run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" +run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.default_dbc_suffix | default('-dl-default') }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 3f28e8db..42e53664 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -173,11 +173,10 @@ run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}" config: name: "{{ __dw_config.name | default('') }}" load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" - use_default_dbc: "{{ use_default_dbc }}" + use_default_dbc: "{{ __dw_config.name is undefined | bool }}" virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" loop: "{{ run__dw_definitions }}" loop_control: @@ -189,19 +188,20 @@ run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: config: - name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" - dbc_name: "{{ item.0.name }}" - use_default_dbc: "{{ item.0.use_default_dbc }}" - type: "{{ item.1.type | default('hive') }}" - template: "{{ item.1.template | default('xsmall') }}" - autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" - autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}" - common_configs: "{{ item.1.configs.common_configs | default({}) }}" - application_configs: "{{ item.1.configs.application_configs | default({}) }}" - ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" - enable_sso: "{{ item.1.configs.enable_sso | default(None) }}" - tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}" + name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + dbc_name: "{{ __dw_config.0.name }}" + use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" + type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" + template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" + autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}" + autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}" + common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}" + application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}" + ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}" + enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}" + tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: + loop_var: __dw_config index_var: __dw_dbc_index - label: "{{ item.0.name }}" \ No newline at end of file + label: "{{ __dw_config.0.name }}" \ No newline at end of file diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index b32fd168..3d691b74 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -69,6 +69,19 @@ env: "{{ run__env_name }}" register: run__dw_list + - name: Fetch all the Database Catalogs under the cluster + cloudera.cloud.dw_database_catalog_info: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + register: __dbc_list + + - name: Set default Database catalog id for the cluster + when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$') + ansible.builtin.set_fact: + __default_dbc_id: "{{ __dw_dbc.id }}" + loop: "{{ __dbc_list.database_catalogs }}" + loop_control: + loop_var: __dw_dbc + - name: Create CDP DW Database catalogs when: not __dw_dbc_config.use_default_dbc cloudera.cloud.dw_database_catalog: @@ -99,9 +112,9 @@ label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map - when: __dw_dbc_build_async.database_catalogs is defined + when: __dw_dbc_build_async.database_catalog is defined ansible.builtin.set_fact: - run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}" + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async @@ -109,12 +122,12 @@ - name: Create CDP DW Virtual warehouse cloudera.cloud.dw_virtual_warehouse: cluster_id: "{{ run__dw_list.clusters[0].id }}" - dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" + dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}" type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}" - autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}" common_configs: "{{ __dw_vw_config.common_configs }}" application_configs: "{{ __dw_vw_config.application_configs }}" ldap_groups: "{{ __dw_vw_config.ldap_groups }}" From 76679377b6a7916dd2485adac530d327afafad2f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:26 -0500 Subject: [PATCH 08/34] Fix missing defaults for DW overlay network and private worker nodes Signed-off-by: Webster Mudge --- roles/runtime/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 2ef56cc5..677a030f 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -74,9 +74,9 @@ run__dw_definitions: "{{ dw.definitions | default([{}]) }}" run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" run__dw_tags: "{{ dw.tags | default(common__tags) }}" -run__dw_overlay_network: "{{ dw.overlay_network | bool }}" +run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bool }}" run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" -run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" From 2694f5634cd9bc3e89f17c9de61b0e7c3dd6d680 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:26 -0500 Subject: [PATCH 09/34] Add DW configuration parameters to documentation Signed-off-by: Webster Mudge --- docs/configuration.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/configuration.yml b/docs/configuration.yml index 71e5410e..c908da21 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -35,7 +35,33 @@ df: terminate_deployments: dw: definitions: + - name: + use_default_dbc: + load_demo_data: + virtual_warehouses: + - name: + vw_type: + template: + autoscaling: + min_cluster: + max_cluster: + tags: + configs: + common_configs: + application_configs: + enable_sso: + ldap_groups: suffix: + vw: + suffix: + tags: + overlay_network: + private_load_balancer: + private_worker_nodes: + force_delete: + default_vw_type: + default_template_type: + default_dbc_suffix: env: aws: policy: From cf4579bbbc6e0bc5ae212f50f64129ef5088829b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:26 -0500 Subject: [PATCH 10/34] Fix configuration parameter names Signed-off-by: Webster Mudge --- docs/configuration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.yml b/docs/configuration.yml index c908da21..f64d7e56 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -40,11 +40,11 @@ dw: load_demo_data: virtual_warehouses: - name: - vw_type: + type: template: autoscaling: - min_cluster: - max_cluster: + min_nodes: + max_nodes: tags: configs: common_configs: From 498edca253d682e2b633d0c691df50a00cd5cf04 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:26 -0500 Subject: [PATCH 11/34] Refactor to use 'omit' for DW config and setup Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 42e53664..21236e1f 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -193,12 +193,12 @@ use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" - autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}" - autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}" - common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}" - application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}" - ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}" - enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}" + autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(omit) }}" + autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(omit) }}" + common_configs: "{{ __dw_config.1.configs.common_configs | default(omit) }}" + application_configs: "{{ __dw_config.1.configs.application_configs | default(omit) }}" + ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(omit) }}" + enable_sso: "{{ __dw_config.1.configs.enable_sso | default(omit) }}" tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: From c41cfdb78c29ac9e95bb8e23a6fe2bf5c3f3fdfd Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 12/34] Remove extraneous DW cluster setup wait Signed-off-by: Webster Mudge --- roles/runtime/tasks/setup_aws.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 3d691b74..6e02998d 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -53,12 +53,9 @@ register: __dw_builds - name: Wait for CDP DW cluster setup to complete + when: false ansible.builtin.async_status: jid: "{{ __dw_builds.ansible_job_id }}" - #loop_control: - # loop_var: __opdb_build - # label: "{{ __opdb_build.__opdb_config.name }}" - #loop: "{{ __opdb_builds.results }}" register: __dw_builds_async until: __dw_builds_async.finished retries: 120 From a8e5bdc3dff07967e641bfb2c4794342972b1013 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 13/34] Refactor DW VM module defaults and omit logic Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 8 ++------ roles/runtime/tasks/setup_aws.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 21236e1f..25df91af 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -193,13 +193,9 @@ use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" - autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(omit) }}" - autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(omit) }}" - common_configs: "{{ __dw_config.1.configs.common_configs | default(omit) }}" - application_configs: "{{ __dw_config.1.configs.application_configs | default(omit) }}" - ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(omit) }}" - enable_sso: "{{ __dw_config.1.configs.enable_sso | default(omit) }}" tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" + autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}" + configs: "{{ __dw_config.1.configs | default({}) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: loop_var: __dw_config diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 6e02998d..f99ae0b7 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -123,13 +123,13 @@ type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}" - autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}" - common_configs: "{{ __dw_vw_config.common_configs }}" - application_configs: "{{ __dw_vw_config.application_configs }}" - ldap_groups: "{{ __dw_vw_config.ldap_groups }}" - enable_sso: "{{ __dw_vw_config.enable_sso | bool }}" tags: "{{ __dw_vw_config.tags }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling.min_nodes | default(omit) }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling.max_nodes | default(omit) }}" + common_configs: "{{ __dw_vw_config.configs.common_configs | default(omit) }}" + application_configs: "{{ __dw_vw_config.configs.application_configs | default(omit) }}" + ldap_groups: "{{ __dw_vw_config.configs.ldap_groups | default(omit) }}" + enable_sso: "{{ __dw_vw_config.configs.enable_sso | default(omit) }}" wait: yes async: 3600 # 1 hour timeout poll: 0 From 839f4cd26548219dbbed9ef636f2b561cd501f57 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 14/34] Add missing force_delete and force_teardown parameters Signed-off-by: Webster Mudge --- docs/configuration.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/configuration.yml b/docs/configuration.yml index f64d7e56..79cf1f1c 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -14,6 +14,7 @@ datahub: gcp: suffix: tags: + force_delete: de: definitions: suffix: @@ -32,7 +33,6 @@ df: teardown: persist: force_delete: - terminate_deployments: dw: definitions: - name: @@ -221,7 +221,7 @@ globals: dynamic_inventory: vm: count: - os: + force_teardown: gcloud_credential_file: infra_type: labels: @@ -379,6 +379,7 @@ ml: suffix: tags: public_loadbalancer: + force_delete: opdb: definitions: suffix: From e34cad2194ba82e57c72fb57249026796bd15877 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 15/34] Move defaults into nested configs Signed-off-by: Webster Mudge --- docs/configuration.yml | 8 +++++--- roles/runtime/defaults/main.yml | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/configuration.yml b/docs/configuration.yml index 79cf1f1c..aa1f26f5 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -54,14 +54,16 @@ dw: suffix: vw: suffix: + type: + template: + dbc: + suffix: + default_suffix: tags: overlay_network: private_load_balancer: private_worker_nodes: force_delete: - default_vw_type: - default_template_type: - default_dbc_suffix: env: aws: policy: diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 677a030f..8373aa40 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -78,9 +78,9 @@ run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bo run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" -run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" -run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" -run__dw_default_dbc_suffix: "{{ dw.default_dbc_suffix | default('-dl-default') }}" +run__dw_default_vw_type: "{{ dw.vw.type | default('hive') }}" +run__dw_default_template_type: "{{ dw.vw.template | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.dbc.default_suffix | default('dl-default') }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" From c4c7c734740cefe34f7fd7ec4a593ef285af5e6b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 16/34] Add default for subelements filter Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 25df91af..53bdd051 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -196,7 +196,7 @@ tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}" configs: "{{ __dw_config.1.configs | default({}) }}" - loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" + loop: "{{ run__dw_dbc_configs | default([{}]) | subelements('virtual_warehouses')}}" loop_control: loop_var: __dw_config index_var: __dw_dbc_index From 976a14cd8c5d5624811c5f8d83f77d6461feb09c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:27 -0500 Subject: [PATCH 17/34] Convert DW teardown to async Signed-off-by: Webster Mudge --- roles/runtime/tasks/teardown_base.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index ec5a028e..42cddaec 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -33,7 +33,6 @@ register: __opdb_teardowns_info - name: Execute CDP DW cluster teardown - register: __dw_teardown_info when: - run__include_dw or run__force_teardown | bool - run__env_info.environments | length > 0 @@ -41,8 +40,11 @@ cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent - wait: no + wait: yes force: "{{ run__dw_force_delete }}" + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_cluster_teardown_info - name: Execute CDP Dataflow teardown register: __df_teardown_info @@ -164,16 +166,16 @@ retries: 120 delay: 30 -- name: Wait for CDP DW deployments to decommission +- name: Wait for CDP Data Warehouse deployments to decommission when: - - __dw_teardown_info is defined - - __dw_teardown_info.started | default(False) - cloudera.cloud.dw_cluster: - env: "{{ run__env_name }}" - state: absent - wait: yes - force: "{{ run__dw_force_delete }}" - + - __dw_cluster_teardown_info is defined + ansible.builtin.async_status: + jid: "{{ __dw_cluster_teardown_info.ansible_job_id }}" + register: __dw_cluster_teardown_async + until: __dw_cluster_teardown_async.finished + retries: 120 + delay: 30 + - name: Wait for CDP OpDB deployments to decommission when: - __opdb_teardowns_info is defined From 90833596b18e31bd41289397b95add1d2fd2e937 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:28 -0500 Subject: [PATCH 18/34] Update DBC name logic Add labels Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 53bdd051..e61bcf19 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -64,7 +64,7 @@ - name: Retrieve Image Catalog File ansible.builtin.uri: url: "{{ run__datahub_image_catalog_url }}" - #no_log: yes + no_log: yes register: __datahub_image_catalog - name: Set fact for latest CDP Image in Catalog @@ -174,9 +174,8 @@ vars: include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" config: - name: "{{ __dw_config.name | default('') }}" + name: "{{ __dw_config.name | default(run__dw_default_dbc) }}" load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" - use_default_dbc: "{{ __dw_config.name is undefined | bool }}" virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" loop: "{{ run__dw_definitions }}" loop_control: @@ -188,16 +187,15 @@ run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: config: - name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" dbc_name: "{{ __dw_config.0.name }}" - use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" + name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}" configs: "{{ __dw_config.1.configs | default({}) }}" - loop: "{{ run__dw_dbc_configs | default([{}]) | subelements('virtual_warehouses')}}" + loop: "{{ run__dw_dbc_configs | default({}) | subelements('virtual_warehouses')}}" loop_control: loop_var: __dw_config index_var: __dw_dbc_index - label: "{{ __dw_config.0.name }}" \ No newline at end of file + label: "{{ config.name }}" \ No newline at end of file From 05653ecb2f30bd385f0399fcc409e5bf57b55bba Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:28 -0500 Subject: [PATCH 19/34] Update DW teardown for single service deployment Signed-off-by: Webster Mudge --- roles/runtime/tasks/teardown_base.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index 42cddaec..a827adb4 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -33,18 +33,16 @@ register: __opdb_teardowns_info - name: Execute CDP DW cluster teardown + register: __dw_teardown_info when: - run__include_dw or run__force_teardown | bool - run__env_info.environments | length > 0 - - run__env_info.environments[0].descendants.dw | length > 0 + #- run__env_info.environments[0].descendants.dw | length > 0 cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent - wait: yes + wait: no force: "{{ run__dw_force_delete }}" - async: 3600 # 1 hour timeout - poll: 0 - register: __dw_cluster_teardown_info - name: Execute CDP Dataflow teardown register: __df_teardown_info @@ -166,16 +164,16 @@ retries: 120 delay: 30 -- name: Wait for CDP Data Warehouse deployments to decommission +- name: Wait for CDP DW deployments to decommission when: - - __dw_cluster_teardown_info is defined - ansible.builtin.async_status: - jid: "{{ __dw_cluster_teardown_info.ansible_job_id }}" - register: __dw_cluster_teardown_async - until: __dw_cluster_teardown_async.finished - retries: 120 - delay: 30 - + - __dw_teardown_info is defined + - __dw_teardown_info.started | default(False) + cloudera.cloud.dw_cluster: + env: "{{ run__env_name }}" + state: absent + wait: yes + force: "{{ run__dw_force_delete }}" + - name: Wait for CDP OpDB deployments to decommission when: - __opdb_teardowns_info is defined From 8ad8d1ae95053dda5e74e9bd908f7b7588c482f6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:28 -0500 Subject: [PATCH 20/34] Update for DW Cluster ID retrieval Signed-off-by: Webster Mudge --- roles/runtime/tasks/setup_aws.yml | 50 +++++++------------------------ 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index f99ae0b7..5f91a01e 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -35,7 +35,7 @@ label: "{{ __aws_instance_item.instance_id }}" loop: "{{ __aws_instance_info.instances }}" -- name: Setup CDP DW cluster on AWS +- name: Setup CDP DW Cluster on AWS when: run__include_dw tags: dw block: @@ -48,41 +48,12 @@ aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present wait: yes - async: 3600 # 1 hour timeout - poll: 0 register: __dw_builds - - name: Wait for CDP DW cluster setup to complete - when: false - ansible.builtin.async_status: - jid: "{{ __dw_builds.ansible_job_id }}" - register: __dw_builds_async - until: __dw_builds_async.finished - retries: 120 - delay: 30 - - - name: Retrieve CDP DW experiences - cloudera.cloud.dw_cluster_info: - env: "{{ run__env_name }}" - register: run__dw_list - - - name: Fetch all the Database Catalogs under the cluster - cloudera.cloud.dw_database_catalog_info: - cluster_id: "{{ run__dw_list.clusters[0].id }}" - register: __dbc_list - - - name: Set default Database catalog id for the cluster - when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$') - ansible.builtin.set_fact: - __default_dbc_id: "{{ __dw_dbc.id }}" - loop: "{{ __dbc_list.database_catalogs }}" - loop_control: - loop_var: __dw_dbc - - - name: Create CDP DW Database catalogs - when: not __dw_dbc_config.use_default_dbc + # TODO Refactor for non-AWS? + - name: Create CDP DW Database Catalogs cloudera.cloud.dw_database_catalog: - cluster_id : "{{ run__dw_list.clusters[0].id }}" + cluster_id: "{{ __dw_builds.cluster.id }}" name: "{{ __dw_dbc_config.name }}" load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" state: present @@ -95,8 +66,7 @@ label: "{{ __dw_dbc_config.name }}" register: __dw_dbc_builds - - name: Wait for CDP DW Database catalogs setup to complete - when: __dw_dbc_build.ansible_job_id is defined + - name: Wait for CDP DW Database Catalogs setup to complete ansible.builtin.async_status: jid: "{{ __dw_dbc_build.ansible_job_id }}" register: __dw_dbc_builds_async @@ -108,18 +78,19 @@ loop_var: __dw_dbc_build label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - - name: Set CDP DW Database catalog name to id map - when: __dw_dbc_build_async.database_catalog is defined + - name: Set CDP DW Database Catalog ID map ansible.builtin.set_fact: run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async + label: "{{ __dw_dbc_build_async.database_catalog.name }}" + failed_when: __dw_dbc_build_async.database_catalog is undefined - name: Create CDP DW Virtual warehouse cloudera.cloud.dw_virtual_warehouse: - cluster_id: "{{ run__dw_list.clusters[0].id }}" - dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}" + cluster_id: "{{ __dw_builds.cluster.id }}" + dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] }}" type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" @@ -140,7 +111,6 @@ label: "{{ __dw_vw_config.name }}" - name: Wait for CDP DW Virtual warehouse setup to complete - when: __dw_vw_build.ansible_job_id is defined ansible.builtin.async_status: jid: "{{ __dw_vw_build.ansible_job_id }}" register: __dw_vw_builds_async From 382701fbc5b6862cac9981b96c14c1d63696849d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:28 -0500 Subject: [PATCH 21/34] Update for monitoring DW service teardown Signed-off-by: Webster Mudge --- roles/runtime/tasks/teardown_base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index a827adb4..2998c638 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -167,7 +167,7 @@ - name: Wait for CDP DW deployments to decommission when: - __dw_teardown_info is defined - - __dw_teardown_info.started | default(False) + # - __dw_teardown_info.started | default(False) cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent From 63235aa01fa15c6106b7fd276b5f9511e0124aab Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:28 -0500 Subject: [PATCH 22/34] Update DW defaults and add missing DBC default name Signed-off-by: Webster Mudge --- roles/runtime/defaults/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 8373aa40..b900c25b 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -78,9 +78,10 @@ run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bo run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" -run__dw_default_vw_type: "{{ dw.vw.type | default('hive') }}" -run__dw_default_template_type: "{{ dw.vw.template | default('xsmall') }}" -run__dw_default_dbc_suffix: "{{ dw.dbc.default_suffix | default('dl-default') }}" +run__dw_default_vw_type: "{{ dw.default_vw.type | default('hive') }}" +run__dw_default_template_type: "{{ dw.default_template.type | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.default_dbc.suffix | default('dl-default') }}" +run__dw_default_dbc: "{{ dw.default_dbc.name | default([run__env_name, run__dw_default_dbc_suffix] | join('-')) }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" From 294406e2d62e0ab51b62ba12c5dcc267e154441f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 23/34] Uplift AWS VPC and subnet IDs to common Signed-off-by: Webster Mudge --- roles/common/defaults/main.yml | 6 ++++++ roles/infrastructure/defaults/main.yml | 6 +++--- roles/platform/defaults/main.yml | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 81b0a69f..18dc4595 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -93,11 +93,17 @@ common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_sub common__aws_region: "{{ infra.aws.region | default('eu-west-1') }}" common__aws_profile: "{{ infra.aws.profile | default('') }}" common__aws_role_suffix: "{{ infra.aws.role.suffix | default(common__role_suffix) }}" + common__aws_datalake_admin_role_name: "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}" common__aws_datalake_admin_suffix: "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}" common__aws_idbroker_role_name: "{{ env.aws.role.name.idbroker | default([common__namespace, common__aws_idbroker_suffix, common__aws_role_suffix] | join('-')) }}" common__aws_idbroker_suffix: "{{ env.aws.role.label.idbroker | default(common__idbroker_suffix) }}" + +common__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | default('') }}" +common__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" +common__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" + # Azure Infra common__azure_storage_name: "{{ infra.azure.storage.name | default(common__storage_name | replace('-','')) }}" diff --git a/roles/infrastructure/defaults/main.yml b/roles/infrastructure/defaults/main.yml index 9dfa6f0c..c64d95e9 100644 --- a/roles/infrastructure/defaults/main.yml +++ b/roles/infrastructure/defaults/main.yml @@ -83,9 +83,9 @@ infra__vpc_user_ports: "{{ infra.vpc.user_ports | default([infra__a infra__vpc_user_cidr: "{{ infra.vpc.user_cidr | default([]) }}" infra__vpc_tunneled_cidr: "{{ infra.vpc.tunneled_cidr | default([]) }}" -infra__aws_vpc_id: "{{ infra.aws.vpc.existing.vpc_id | default('') }}" -infra__aws_public_subnet_ids: "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}" -infra__aws_private_subnet_ids: "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}" +infra__aws_vpc_id: "{{ common__aws_vpc_id }}" +infra__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" +infra__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" infra__security_group_knox_name: "{{ common__security_group_knox_name }}" infra__security_group_default_name: "{{ common__security_group_default_name }}" diff --git a/roles/platform/defaults/main.yml b/roles/platform/defaults/main.yml index 2d5d4421..305b1ac9 100644 --- a/roles/platform/defaults/main.yml +++ b/roles/platform/defaults/main.yml @@ -103,6 +103,9 @@ plat__aws_storage_suffix: "{{ env.aws.storage.suffix | defau plat__aws_role_tags: "{{ env.aws.role.tags | default({}) }}" plat__aws_policy_tags: "{{ env.aws.policy.tags | default({}) }}" plat__aws_storage_tags: "{{ env.aws.storage.tags | default({}) }}" +plat__aws_vpc_id: "{{ common__aws_vpc_id }}" +plat__aws_public_subnet_ids: "{{ common__aws_public_subnet_ids }}" +plat__aws_private_subnet_ids: "{{ common__aws_private_subnet_ids }}" plat__aws_xaccount_suffix: "{{ env.aws.role.label.cross_account | default(common__xaccount_suffix) }}" plat__aws_idbroker_suffix: "{{ common__aws_idbroker_suffix }}" From 923fa096cd2e39211660e8b46f751d097c5c9284 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 24/34] Add namespace and role names Signed-off-by: Webster Mudge --- roles/common/meta/main.yml | 2 ++ roles/platform/meta/main.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/roles/common/meta/main.yml b/roles/common/meta/main.yml index 90bf17c6..acb23b1a 100644 --- a/roles/common/meta/main.yml +++ b/roles/common/meta/main.yml @@ -13,6 +13,8 @@ # limitations under the License. galaxy_info: + role_name: platform + namespace: cloudera author: Webster Mudge (wmudge@cloudera.com) description: > Shared configuration variables managed by role dependency. diff --git a/roles/platform/meta/main.yml b/roles/platform/meta/main.yml index f530ab15..2045f3f1 100644 --- a/roles/platform/meta/main.yml +++ b/roles/platform/meta/main.yml @@ -13,6 +13,8 @@ # limitations under the License. galaxy_info: + role_name: platform + namespace: cloudera author: Webster Mudge (wmudge@cloudera.com) description: > Deployment and management of Cloudera Data Platform (CDP) Public Cloud core From 5daf8b8219b2bb5f1111206ad35845fd877810c8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 25/34] Add AWS SSH key check Signed-off-by: Webster Mudge --- roles/platform/tasks/initialize_aws.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/platform/tasks/initialize_aws.yml b/roles/platform/tasks/initialize_aws.yml index 568da827..bf835d48 100644 --- a/roles/platform/tasks/initialize_aws.yml +++ b/roles/platform/tasks/initialize_aws.yml @@ -17,4 +17,9 @@ - name: Retrieve AWS Caller details amazon.aws.aws_caller_info: register: __aws_caller_info - failed_when: __aws_caller_info.account is not defined \ No newline at end of file + failed_when: __aws_caller_info.account is not defined + +- name: Confirm AWS SSH Public Key ID exists + ansible.builtin.command: aws ec2 describe-key-pairs --region "{{ plat__region }}" --key-name "{{ plat__public_key_id }}" + register: __aws_ssh_key_pair + failed_when: __aws_ssh_key_pair.rc != 0 \ No newline at end of file From a75bbad559c361b104a687aac5ee0f2a5a18e27b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 26/34] Convert public endpoint access scheme to ternary Signed-off-by: Webster Mudge --- roles/platform/tasks/setup_aws_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/platform/tasks/setup_aws_env.yml b/roles/platform/tasks/setup_aws_env.yml index 61378d9a..a93ebdc7 100644 --- a/roles/platform/tasks/setup_aws_env.yml +++ b/roles/platform/tasks/setup_aws_env.yml @@ -31,7 +31,7 @@ subnet_ids: "{{ plat__aws_public_subnet_ids | union(plat__aws_private_subnet_ids) }}" tags: "{{ plat__tags }}" tunnel: "{{ plat__tunnel }}" - endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}" + endpoint_access_scheme: "{{ plat__public_endpoint_access | ternary('PUBLIC', omit) }}" endpoint_access_subnets: "{{ plat__aws_public_subnet_ids | default(omit) }}" freeipa: instanceCountByGroup: "{{ plat__env_freeipa }}" \ No newline at end of file From b6cd0d0176f4e078bb56ff463f23e871f5e621c4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 27/34] Streamline public and private subnet ID assignment Signed-off-by: Webster Mudge --- roles/infrastructure/tasks/initialize_aws.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/infrastructure/tasks/initialize_aws.yml b/roles/infrastructure/tasks/initialize_aws.yml index 7db6114f..c80d3bc9 100644 --- a/roles/infrastructure/tasks/initialize_aws.yml +++ b/roles/infrastructure/tasks/initialize_aws.yml @@ -115,7 +115,6 @@ - name: Set facts for existing AWS Public Subnet IDs ansible.builtin.set_fact: - infra__aws_public_subnet_ids: "{{ infra__aws_public_subnet_ids }}" infra__aws_subnet_ids: "{{ infra__aws_subnet_ids | default([]) | union(infra__aws_public_subnet_ids) }}" infra__aws_vpc_id: "{{ __aws_public_subnets_info.subnets | map(attribute='vpc_id') | list | first }}" From 1bc90e1c29946359f0ce866d889b998080522289 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:29 -0500 Subject: [PATCH 28/34] Update AWS VPC and subnet ID assignment to support runlevel, then upstream, and finally discovery Signed-off-by: Webster Mudge --- roles/platform/tasks/initialize_setup_aws.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/platform/tasks/initialize_setup_aws.yml b/roles/platform/tasks/initialize_setup_aws.yml index 9f4b6112..53cf04a4 100644 --- a/roles/platform/tasks/initialize_setup_aws.yml +++ b/roles/platform/tasks/initialize_setup_aws.yml @@ -48,7 +48,7 @@ plat__aws_xaccount_account_id: "{{ plat__cdp_xaccount_account_id }}" # Runlevel first, upstream second, and discover third -- name: Discover AWS VPC if not defined +- name: Discover AWS VPC if not defined or established by Infrastructure when: plat__aws_vpc_id == "" and infra__aws_vpc_id is undefined block: - name: Query AWS VPC by name @@ -70,13 +70,12 @@ plat__aws_vpc_id: "{{ __aws_vpc_info.vpcs[0].id }}" - name: Set fact for AWS VPC ID if established by Infrastructure - when: infra__aws_vpc_id is defined + when: plat__aws_vpc_id == "" and infra__aws_vpc_id is defined ansible.builtin.set_fact: plat__aws_vpc_id: "{{ infra__aws_vpc_id }}" -# Runlevel first, upstream second, and discover third -- name: Handle AWS Public and Private VPC Subnets if not defined - when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids +- name: Handle AWS Subnet IDs if not defined + when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids # Defaults are empty lists block: - name: Query AWS Subnets amazon.aws.ec2_vpc_subnet_info: @@ -143,6 +142,7 @@ ansible.builtin.set_fact: plat__endpoint_access_scheme: "PUBLIC" +# TODO Collapse the two SG queries together - name: Discover AWS Security Group for Knox when: infra__aws_security_group_knox_id is undefined block: @@ -150,6 +150,7 @@ amazon.aws.ec2_group_info: region: "{{ plat__region }}" filters: + vpc-id: "{{ plat__aws_vpc_id }}" group-name: "{{ plat__security_group_knox_name }}" register: __aws_security_group_knox_info @@ -170,6 +171,7 @@ amazon.aws.ec2_group_info: region: "{{ plat__region }}" filters: + vpc-id: "{{ plat__aws_vpc_id }}" group-name: "{{ plat__security_group_default_name }}" register: __aws_security_group_default_info From f926a15faf80079d5826e7b342902fd95260b052 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:30 -0500 Subject: [PATCH 29/34] Check DW Virtual Warehouse tags Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index e61bcf19..f0e83da6 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -52,6 +52,7 @@ ansible.builtin.set_fact: run__cdp_datalake_version: "{{ __cdp_datalake_version_info.versions[0].runtimeVersion | trim }}" +# TODO Discover version if upstream is not present - name: Set fact for CDP Datalake version by assignment when: plat__cdp_datalake_version is defined ansible.builtin.set_fact: @@ -168,7 +169,7 @@ - name: Prepare for CDP DW experiences when: run__include_dw block: - - name: Construct CDP DW Data catalog configurations + - name: Construct CDP DW Data Catalog configurations ansible.builtin.set_fact: run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: @@ -182,7 +183,7 @@ loop_var: __dw_config index_var: __dw_config_index - - name: Construct CDP DW Virtual warehouse configurations + - name: Construct CDP DW Virtual Warehouse configurations ansible.builtin.set_fact: run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: @@ -198,4 +199,18 @@ loop_control: loop_var: __dw_config index_var: __dw_dbc_index - label: "{{ config.name }}" \ No newline at end of file + label: "{{ config.name }}" + + - name: Check CDP DW Virtual Warehouse tags + ansible.builtin.assert: + that: + - __dw_vw_config.tags | dict2items | rejectattr('value', 'regex', '[^-_a-zA-Z0-9.=:+@]+') | list + fail_msg: + - "A tag in Data Warehouse, '{{ __dw_vw_config.name }}', does not meet requirements;" + - "current tags: {{ __dw_vw_config.tags}}." + - "Allowed characters in tags are letters, numbers and the following characters: _.:/=+-@" + quiet: yes + loop_control: + loop_var: __dw_vw_config + label: "{{ __dw_vw_config.name }}" + loop: "{{ run__dw_vw_configs }}" From 2dfb06747eb8037e43c7994895cb872b356b8585 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:30 -0500 Subject: [PATCH 30/34] Check public subnet count for public load balancer Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_setup.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roles/runtime/tasks/initialize_setup.yml b/roles/runtime/tasks/initialize_setup.yml index 40bc06a6..171185cb 100644 --- a/roles/runtime/tasks/initialize_setup.yml +++ b/roles/runtime/tasks/initialize_setup.yml @@ -29,3 +29,19 @@ - dw - opdb - dh + +- name: Prepare for CDP DW experiences + when: run__include_dw + block: + - name: Confirm public subnet count CDP DW public load balancer + when: not run__dw_private_load_balancer + ansible.builtin.assert: + that: + - run__public_subnet_ids | length == 3 + fail_msg: "Must have exactly 3 public subnets when deploying CDP Data Warehouse with a public load balancer" + quiet: yes + tags: + - ml + - dw + - opdb + - dh From f5f8a94da78c122abf9facd862691a12ac9b8ea0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:30 -0500 Subject: [PATCH 31/34] Discover DW deployments for teardown Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_teardown.yml | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/roles/runtime/tasks/initialize_teardown.yml b/roles/runtime/tasks/initialize_teardown.yml index 2baaba83..dce72dbf 100644 --- a/roles/runtime/tasks/initialize_teardown.yml +++ b/roles/runtime/tasks/initialize_teardown.yml @@ -24,11 +24,37 @@ when: not run__force_teardown ansible.builtin.include_tasks: "initialize_base.yml" -- name: Discover CDP DF Deployments - register: run__df_service_info +- name: Discover CDP Dataflow deployments when: run__include_df cloudera.cloud.df_info: name: "{{ run__env_name }}" + register: run__df_env_info + +- name: Discover CDP Data Warehouse deployments + when: + - run__include_dw + - not run__force_teardown | bool or not run__dw_force_delete | bool + block: + - name: Discover CDP Data Warehouse cluster + cloudera.cloud.dw_cluster_info: + env: "{{ run__env_name }}" + register: __dw_list + + - name: Initialize CDP Data Warehouse cluster id + ansible.builtin.set_fact: + __dw_cluster_id: "{{ __dw_list.clusters | map(attribute='id') | first | default(omit) }}" + + - name: Discover CDP Data Warehouse database catalogs + when: __dw_cluster_id is defined + cloudera.cloud.dw_database_catalog_info: + cluster_id: "{{ __dw_cluster_id }}" + register: __dw_dbc_list + + - name: Discover CDP Data Warehouse virtual warehouses + when: __dw_cluster_id is defined + cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: "{{ __dw_cluster_id }}" + register: __dw_vw_list - name: Initialize Purge of all Runtimes in Environment when: From 67aebeba0605482829d7fa401104b407983c7a7c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:30 -0500 Subject: [PATCH 32/34] Update DW cluster setup (remove initial catalog setup) Signed-off-by: Webster Mudge --- roles/runtime/tasks/setup_aws.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 5f91a01e..9a5605dc 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -44,18 +44,8 @@ env: "{{ run__env_name }}" overlay: "{{ run__dw_overlay_network }}" private_load_balancer: "{{ run__dw_private_load_balancer }}" - aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}" - state: present - wait: yes - register: __dw_builds - - # TODO Refactor for non-AWS? - - name: Create CDP DW Database Catalogs - cloudera.cloud.dw_database_catalog: - cluster_id: "{{ __dw_builds.cluster.id }}" - name: "{{ __dw_dbc_config.name }}" - load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" + aws_public_subnets: "{{ run__aws_public_subnet_ids }}" + aws_private_subnets: "{{ run__aws_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present wait: yes async: 3600 # 1 hour timeout From 18eac0e1cd103f0f7f0705fa2f111c2a4134048e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:30 -0500 Subject: [PATCH 33/34] Rename 'df' module to 'df_service' Signed-off-by: Webster Mudge --- roles/runtime/tasks/setup_base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index a1991f6d..b81a9323 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -109,7 +109,7 @@ when: run__include_df tags: df cloudera.cloud.df_service: - env_crn: "{{ run__cdp_env_crn }}" + name: "{{ run__cdp_env_crn }}" nodes_min: "{{ run__df_nodes_min }}" nodes_max: "{{ run__df_nodes_max }}" public_loadbalancer: "{{ run__df_public_loadbalancer }}" @@ -179,7 +179,7 @@ when: run__include_df tags: df cloudera.cloud.df_service: - env_crn: "{{ run__cdp_env_crn }}" + name: "{{ run__cdp_env_crn }}" wait: yes - name: Create CDP DE Virtual clusters From d45fe7964436ca0bed7db14ded7393fd93ced7ab Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 24 Nov 2021 12:13:31 -0500 Subject: [PATCH 34/34] Add expanded DW teardown options Signed-off-by: Webster Mudge --- roles/runtime/tasks/teardown_base.yml | 120 +++++++++++++++++++++----- 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/roles/runtime/tasks/teardown_base.yml b/roles/runtime/tasks/teardown_base.yml index 2998c638..9ac98e05 100644 --- a/roles/runtime/tasks/teardown_base.yml +++ b/roles/runtime/tasks/teardown_base.yml @@ -32,26 +32,92 @@ poll: 0 register: __opdb_teardowns_info -- name: Execute CDP DW cluster teardown - register: __dw_teardown_info +- name: Execute CDP Data Warehouse services teardown when: - - run__include_dw or run__force_teardown | bool + - run__include_dw + - not run__force_teardown | bool and not run__dw_force_delete | bool - run__env_info.environments | length > 0 - #- run__env_info.environments[0].descendants.dw | length > 0 - cloudera.cloud.dw_cluster: - env: "{{ run__env_name }}" - state: absent - wait: no - force: "{{ run__dw_force_delete }}" + block: + - name: Execute CDP Data Warehouse virtual warehouse teardown + cloudera.cloud.dw_virtual_warehouse: + cluster_id: "{{ __dw_cluster_id }}" + id: "{{ __vw_id.id }}" + state: absent + wait: yes + loop: "{{ __dw_vw_list.virtual_warehouses | default([]) }}" + loop_control: + loop_var: __vw_id + label: "{{ __vw_id.name }}" + register: __dw_vw_teardown_info + #ignore_errors: yes + async: 3600 # 1 hour timeout + poll: 0 + + - name: Wait for CDP Data Warehouse virtual warehouses to decommission + ansible.builtin.async_status: + jid: "{{ __dw_vw_teardown_item.ansible_job_id }}" + loop_control: + loop_var: __dw_vw_teardown_item + label: "{{ __dw_vw_teardown_item.__vw_id.name }}" + loop: "{{ __dw_vw_teardown_info.results | default([]) }}" + register: __dw_vw_teardowns_async + until: __dw_vw_teardowns_async.finished + #ignore_errors: yes + retries: 30 + delay: 10 + + rescue: + - debug: + var: ansible_failed_result + + - name: Execute CDP Data Warehouse virtual warehouse teardown (compactor) + cloudera.cloud.dw_virtual_warehouse: + cluster_id: "{{ __dw_cluster_id }}" + id: "{{ __dw_vw_compactor_item.id }}" + state: absent + wait: yes + #loop: "{{ __dw_vw_teardown_info.results | map(attribute='__vw_id') | list }}" + loop: "{{ ansible_failed_result.results | map(attribute='__vw_id') | list }}" + loop_control: + loop_var: __dw_vw_compactor_item + label: "{{ __dw_vw_compactor_item.name }}" + register: __dw_vw_compactor_info + async: 3600 # 1 hour timeout + poll: 0 + + - name: Wait for CDP Data Warehouse virtual warehouses (compactor) to decommission + ansible.builtin.async_status: + jid: "{{ __dw_vw_compactor_teardown_item.ansible_job_id }}" + loop_control: + loop_var: __dw_vw_compactor_teardown_item + label: "{{ __dw_vw_compactor_teardown_item.__dw_vw_compactor_item.name }}" + loop: "{{ __dw_vw_compactor_info.results }}" + register: __dw_vw_compactor_teardowns_async + until: __dw_vw_compactor_teardowns_async.finished + retries: 30 + delay: 10 + + - name: Execute CDP Data Warehouse database catalog teardown + cloudera.cloud.dw_database_catalog: + cluster_id: "{{ __dw_cluster_id }}" + id: "{{ __dbc_id.id }}" + state: absent + wait: yes + loop: "{{ __dw_dbc_list.database_catalogs }}" + loop_control: + loop_var: __dbc_id + label: "{{ __dbc_id.name }}" + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_dbc_teardown_info - name: Execute CDP Dataflow teardown - register: __df_teardown_info when: - run__include_df or run__force_teardown | bool - - run__df_service_info is defined and run__df_service_info.services is defined - - run__df_service_info.services | length > 0 + - run__df_env_info is defined and run__df_env_info.services is defined + - run__df_env_info.services | length > 0 cloudera.cloud.df_service: - df_crn: "{{ __df_teardown_req_item.crn }}" + name: "{{ __df_teardown_req_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" terminate: "{{ run__df_terminate_deployments }}" @@ -59,7 +125,8 @@ wait: no loop_control: loop_var: __df_teardown_req_item - loop: "{{ run__df_service_info.services }}" + loop: "{{ run__df_env_info.services }}" + register: __df_teardown_info - name: Execute CDP ML Workspace teardown when: @@ -164,15 +231,28 @@ retries: 120 delay: 30 -- name: Wait for CDP DW deployments to decommission +- name: Wait for CDP Data Warehouse deployments to decommission + when: __dw_dbc_teardown_info is defined and __dw_dbc_teardown_info.results is defined and not __dw_dbc_teardown_item.skipped + ansible.builtin.async_status: + jid: "{{ __dw_dbc_teardown_item.ansible_job_id }}" + loop_control: + loop_var: __dw_dbc_teardown_item + label: "{{ __dw_dbc_teardown_item.__dbc_id.name }}" + loop: "{{ __dw_dbc_teardown_info.results }}" + register: __dw_dbc_teardowns_async + until: __dw_dbc_teardowns_async.finished + retries: 120 + delay: 30 + +- name: Execute CDP Data Warehouse cluster teardown when: - - __dw_teardown_info is defined - # - __dw_teardown_info.started | default(False) + - run__include_dw or run__force_teardown | bool + - run__env_info.environments | length > 0 cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" state: absent wait: yes - force: "{{ run__dw_force_delete }}" + force: "{{ run__dw_force_delete or run__force_teardown }}" - name: Wait for CDP OpDB deployments to decommission when: @@ -193,9 +273,9 @@ - name: Wait for CDP Dataflow deployment to decommission when: - run__include_df - - run__df_service_info.services | length > 0 + - run__df_env_info.services | length > 0 cloudera.cloud.df_service: - df_crn: "{{ __df_teardown_wait_item.crn }}" + name: "{{ __df_teardown_wait_item.crn }}" persist: "{{ run__df_persist }}" force: "{{ run__df_force_delete }}" state: absent