From 245fc71f47bfea353d5c9c50918ad6dcf924e3c1 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:34 -0400 Subject: [PATCH 01/13] Support CDW VW changes Signed-off-by: Saravanan Raju --- roles/common/defaults/main.yml | 2 +- roles/runtime/defaults/main.yml | 7 +- roles/runtime/tasks/initialize_base.yml | 43 ++++++++ roles/runtime/tasks/initialize_setup_aws.yml | 7 +- roles/runtime/tasks/setup_aws.yml | 109 ++++++++++++++++--- 5 files changed, 149 insertions(+), 19 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 5298b5bb..9d31c7c2 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -121,7 +121,7 @@ common__env_name_suffix: "{{ env.suffix | default(common__env_s common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" -common__tunnel: "{{ env.tunnel | default(False) }}" +common__tunnel: "{{ env.tunnel | default(True) }}" common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" common__env_admin_password: "{{ globals.admin_password | mandatory }}" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 8cce89e2..ddbab2bf 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -65,7 +65,12 @@ run__ml_remove_storage: "{{ ml.remove_storage | default (run__force_ run__ml_public_loadbalancer: "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}" run__dw_definitions: "{{ dw.definitions | default([{}]) }}" -run__dw_suffix: "{{ dw.suffix | default('dw') }}" +run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" +run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" +run__dw_tags: "{{ dw.tags | default(common__tags) }}" +run__dw_overlay_network: "{{ dw.overlay_network | bool }}" +run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index e0bc0ade..92166c19 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -161,3 +161,46 @@ loop_control: loop_var: __ml_config label: "{{ config.name }}" + +- name: Prepare for CDP DW experiences + when: run__include_dw + block: + - name: Construct CDP DW Data catalog configurations + ansible.builtin.set_fact: + run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" + vars: + include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" + use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}" + default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}" + config: + name: "{{ __dw_config.name | default(default_dbc_name) }}" + load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" + use_default_dbc: "{{ use_default_dbc }}" + virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" + loop: "{{ run__dw_definitions }}" + loop_control: + loop_var: __dw_config + index_var: __dw_config_index + + - name: Construct CDP DW Virtual warehouse configurations + ansible.builtin.set_fact: + run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" + vars: + dbc_name: "{{ item.0.name }}" + use_default_dbc: "{{ item.0.use_default_dbc }}" + config: + name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + dbc_name: "{{ dbc_name }}" + use_default_dbc: "{{ use_default_dbc }}" + vw_type: "{{ item.1.vw_type | default('hive') }}" + template: "{{ item.1.template | default('xsmall') }}" + autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}" + autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}" + common_configs: "{{ item.1.configs.common_configs | default({}) }}" + application_configs: "{{ item.1.configs.application_configs | default({}) }}" + ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" + enable_sso: "{{ item.1.configs.enable_sso | default(None) }}" + tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}" + loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" + loop_control: + index_var: __dw_dbc_index diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml index 69d417a2..5473311a 100644 --- a/roles/runtime/tasks/initialize_setup_aws.yml +++ b/roles/runtime/tasks/initialize_setup_aws.yml @@ -49,10 +49,15 @@ label: "{{ __aws_private_subnet_item.subnet_id }}" loop: "{{ __aws_private_subnets_info.subnets }}" + - name: Assign AWS Private Subnet IDs in not assinged + when: run__datahub_private_subnet_ids is undefined + ansible.builtin.set_fact: + run__datahub_private_subnet_ids: "{{ [] }}" + - name: Set fact for AWS Subnet IDs when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined ansible.builtin.set_fact: - run__datahub_subnet_ids: "{{ run__datahub_public_subnet_ids | default([]) | union(run__datahub_private_subnet_ids) }}" + run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}" - name: Set fact for AWS Subnet IDs by assignment when: infra__aws_subnet_ids diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 2c2132d4..f537ded1 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -37,27 +37,104 @@ - name: Setup CDP DW cluster on AWS when: run__include_dw block: - - name: Execute CDP DW cluster setup - cloudera.cloud.dw_cluster: +# - name: Execute CDP DW cluster setup +# cloudera.cloud.dw_cluster: +# env: "{{ run__env_name }}" +# overlay: "{{ run__dw_overlay_network }}" +# aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" +# aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" +# state: present +# wait: yes +# async: 3600 # 1 hour timeout +# poll: 0 +# register: __dw_builds +# +# - name: Wait for CDP DW cluster setup to complete +# ansible.builtin.async_status: +# jid: "{{ __dw_builds.ansible_job_id }}" +# #loop_control: +# # loop_var: __opdb_build +# # label: "{{ __opdb_build.__opdb_config.name }}" +# #loop: "{{ __opdb_builds.results }}" +# register: __dw_builds_async +# until: __dw_builds_async.finished +# retries: 120 +# delay: 30 + + - name: Retrieve CDP DW experiences + cloudera.cloud.dw_cluster_info: env: "{{ run__env_name }}" - overlay: no - # TODO - Allow direct assignment (will need to coordinate with infra role) - aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + register: run__dw_list + + - name: Create CDP DW Database catalogs + when: not __dw_dbc_config.use_default_dbc + cloudera.cloud.dw_dbc: + cluster_id : "{{ run__dw_list.clusters[0].id }}" + name: "{{ __dw_dbc_config.name }}" + load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" state: present wait: yes async: 3600 # 1 hour timeout poll: 0 - register: __dw_builds + loop: "{{ run__dw_dbc_configs }}" + loop_control: + loop_var: __dw_dbc_config + register: __dw_dbc_builds - - name: Wait for CDP DW cluster setup to complete + - name: Wait for CDP DW Database catalogs setup to complete + when: __dw_dbc_build.ansible_job_id is defined ansible.builtin.async_status: - jid: "{{ __dw_builds.ansible_job_id }}" - #loop_control: - # loop_var: __opdb_build - # label: "{{ __opdb_build.__opdb_config.name }}" - #loop: "{{ __opdb_builds.results }}" - register: __dw_builds_async - until: __dw_builds_async.finished + jid: "{{ __dw_dbc_build.ansible_job_id }}" + register: __dw_dbc_builds_async + until: __dw_dbc_builds_async.finished retries: 120 - delay: 30 \ No newline at end of file + delay: 30 + loop: "{{ __dw_dbc_builds.results }}" + loop_control: + loop_var: __dw_dbc_build + + - name: Set CDP DW Database catalog name to id map + when: __dw_dbc_build_async.dbcs is defined + ansible.builtin.set_fact: + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}" + loop: "{{ __dw_dbc_builds_async.results }}" + loop_control: + loop_var: __dw_dbc_build_async + + - name: Create CDP DW Virtual warehouse + cloudera.cloud.dw_vw: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" + vw_type: "{{ __dw_vw_config.vw_type }}" + name: "{{ __dw_vw_config.name }}" + template: "{{ __dw_vw_config.template }}" + autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}" + autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}" + common_configs: "{{ __dw_vw_config.common_configs }}" + application_configs: "{{ __dw_vw_config.application_configs }}" + ldap_groups: "{{ __dw_vw_config.ldap_groups }}" + enable_sso: "{{ __dw_vw_config.enable_sso | bool }}" + tags: "{{ __dw_vw_config.tags }}" + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_vw_builds + loop: "{{ run__dw_vw_configs }}" + loop_control: + loop_var: __dw_vw_config + + - name: Wait for CDP DW Virtual warehouse setup to complete + when: __dw_vw_build.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __dw_vw_build.ansible_job_id }}" + register: __dw_vw_builds_async + until: __dw_vw_builds_async.finished + retries: 120 + delay: 30 + loop: "{{ __dw_vw_builds.results }}" + loop_control: + loop_var: __dw_vw_build + + - name: Print Virtual Warehouses + ansible.builtin.debug: + msg: "Database Cluster : {{ __dw_vw_builds_async }}" \ No newline at end of file From 8d509abcf39f59072e0ce62759de75fb6cbf7eb4 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:35 -0400 Subject: [PATCH 02/13] Revert unnecessary changes Signed-off-by: Saravanan Raju --- roles/common/defaults/main.yml | 2 +- roles/runtime/tasks/setup_aws.yml | 46 +++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index 9d31c7c2..5298b5bb 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -121,7 +121,7 @@ common__env_name_suffix: "{{ env.suffix | default(common__env_s common__datalake_name: "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}" common__datalake_name_suffix: "{{ env.datalake.suffix | default(common__datalake_suffix) }}" -common__tunnel: "{{ env.tunnel | default(True) }}" +common__tunnel: "{{ env.tunnel | default(False) }}" common__public_endpoint_access: "{{ env.public_endpoint_access | default(not common__tunnel) }}" common__env_admin_password: "{{ globals.admin_password | mandatory }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index f537ded1..ba40fea6 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -37,29 +37,29 @@ - name: Setup CDP DW cluster on AWS when: run__include_dw block: -# - name: Execute CDP DW cluster setup -# cloudera.cloud.dw_cluster: -# env: "{{ run__env_name }}" -# overlay: "{{ run__dw_overlay_network }}" -# aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" -# aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" -# state: present -# wait: yes -# async: 3600 # 1 hour timeout -# poll: 0 -# register: __dw_builds -# -# - name: Wait for CDP DW cluster setup to complete -# ansible.builtin.async_status: -# jid: "{{ __dw_builds.ansible_job_id }}" -# #loop_control: -# # loop_var: __opdb_build -# # label: "{{ __opdb_build.__opdb_config.name }}" -# #loop: "{{ __opdb_builds.results }}" -# register: __dw_builds_async -# until: __dw_builds_async.finished -# retries: 120 -# delay: 30 + - name: Execute CDP DW cluster setup + cloudera.cloud.dw_cluster: + env: "{{ run__env_name }}" + overlay: "{{ run__dw_overlay_network }}" + aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" + aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + state: present + wait: yes + async: 3600 # 1 hour timeout + poll: 0 + register: __dw_builds + + - name: Wait for CDP DW cluster setup to complete + ansible.builtin.async_status: + jid: "{{ __dw_builds.ansible_job_id }}" + #loop_control: + # loop_var: __opdb_build + # label: "{{ __opdb_build.__opdb_config.name }}" + #loop: "{{ __opdb_builds.results }}" + register: __dw_builds_async + until: __dw_builds_async.finished + retries: 120 + delay: 30 - name: Retrieve CDP DW experiences cloudera.cloud.dw_cluster_info: From e16dd4d679479a932b88522e1ef9381d6a1789d0 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:35 -0400 Subject: [PATCH 03/13] Add labels for the loops Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 3 ++- roles/runtime/tasks/setup_aws.yml | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 92166c19..09d76cc9 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -189,7 +189,7 @@ dbc_name: "{{ item.0.name }}" use_default_dbc: "{{ item.0.use_default_dbc }}" config: - name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" dbc_name: "{{ dbc_name }}" use_default_dbc: "{{ use_default_dbc }}" vw_type: "{{ item.1.vw_type | default('hive') }}" @@ -204,3 +204,4 @@ loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: index_var: __dw_dbc_index + label: "{{ item.0.name }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index ba40fea6..6651ffda 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -41,8 +41,9 @@ cloudera.cloud.dw_cluster: env: "{{ run__env_name }}" overlay: "{{ run__dw_overlay_network }}" + private_load_balancer: "{{ run__dw_private_load_balancer }}" aws_public_subnets: "{{ run__datahub_public_subnet_ids }}" - aws_private_subnets: "{{ run__datahub_private_subnet_ids }}" + aws_private_subnets: "{{ run__datahub_private_subnet_ids if run__dw_private_worker_nodes else [] }}" state: present wait: yes async: 3600 # 1 hour timeout @@ -79,6 +80,7 @@ loop: "{{ run__dw_dbc_configs }}" loop_control: loop_var: __dw_dbc_config + label: "{{ __dw_dbc_config.name }}" register: __dw_dbc_builds - name: Wait for CDP DW Database catalogs setup to complete @@ -92,6 +94,7 @@ loop: "{{ __dw_dbc_builds.results }}" loop_control: loop_var: __dw_dbc_build + label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map when: __dw_dbc_build_async.dbcs is defined @@ -122,6 +125,7 @@ loop: "{{ run__dw_vw_configs }}" loop_control: loop_var: __dw_vw_config + label: "{{ __dw_vw_config.name }}" - name: Wait for CDP DW Virtual warehouse setup to complete when: __dw_vw_build.ansible_job_id is defined @@ -134,7 +138,4 @@ loop: "{{ __dw_vw_builds.results }}" loop_control: loop_var: __dw_vw_build - - - name: Print Virtual Warehouses - ansible.builtin.debug: - msg: "Database Cluster : {{ __dw_vw_builds_async }}" \ No newline at end of file + label: "{{ __dw_vw_build.__dw_vw_config.name }}" \ No newline at end of file From 54e4186f5a2ec707f61da06718d4f72a077e75f0 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:35 -0400 Subject: [PATCH 04/13] Remove use_default_dbc flag Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 09d76cc9..defdfbad 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -170,10 +170,9 @@ run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}" - default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}" + use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}" config: - name: "{{ __dw_config.name | default(default_dbc_name) }}" + name: "{{ __dw_config.name | default('') }}" load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" use_default_dbc: "{{ use_default_dbc }}" virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" From 700df4b6504755de653e18722a8a8ba6bb83fec3 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:35 -0400 Subject: [PATCH 05/13] Change the vw and dbc module names and other minor refactor Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 6 +++--- roles/runtime/tasks/setup_aws.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index defdfbad..351ff73d 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -191,10 +191,10 @@ name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" dbc_name: "{{ dbc_name }}" use_default_dbc: "{{ use_default_dbc }}" - vw_type: "{{ item.1.vw_type | default('hive') }}" + type: "{{ item.1.type | default('hive') }}" template: "{{ item.1.template | default('xsmall') }}" - autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}" - autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}" + autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" + autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}" common_configs: "{{ item.1.configs.common_configs | default({}) }}" application_configs: "{{ item.1.configs.application_configs | default({}) }}" ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 6651ffda..8ed1edd3 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -69,7 +69,7 @@ - name: Create CDP DW Database catalogs when: not __dw_dbc_config.use_default_dbc - cloudera.cloud.dw_dbc: + cloudera.cloud.dw_database_catalog: cluster_id : "{{ run__dw_list.clusters[0].id }}" name: "{{ __dw_dbc_config.name }}" load_demo_data: "{{ __dw_dbc_config.load_demo_data }}" @@ -97,22 +97,22 @@ label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map - when: __dw_dbc_build_async.dbcs is defined + when: __dw_dbc_build_async.database_catalogs is defined ansible.builtin.set_fact: - run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}" + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async - name: Create CDP DW Virtual warehouse - cloudera.cloud.dw_vw: + cloudera.cloud.dw_virtual_warehouse: cluster_id: "{{ run__dw_list.clusters[0].id }}" dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" - vw_type: "{{ __dw_vw_config.vw_type }}" + type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}" - autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}" common_configs: "{{ __dw_vw_config.common_configs }}" application_configs: "{{ __dw_vw_config.application_configs }}" ldap_groups: "{{ __dw_vw_config.ldap_groups }}" From 3d3a9f89ab82c6fcf14e51c6581908dd2c710c0f Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:35 -0400 Subject: [PATCH 06/13] Minor refactor Signed-off-by: Saravanan Raju --- roles/runtime/tasks/initialize_base.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 351ff73d..41c42a87 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -185,12 +185,10 @@ ansible.builtin.set_fact: run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: - dbc_name: "{{ item.0.name }}" - use_default_dbc: "{{ item.0.use_default_dbc }}" config: name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" - dbc_name: "{{ dbc_name }}" - use_default_dbc: "{{ use_default_dbc }}" + dbc_name: "{{ item.0.name }}" + use_default_dbc: "{{ item.0.use_default_dbc }}" type: "{{ item.1.type | default('hive') }}" template: "{{ item.1.template | default('xsmall') }}" autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" @@ -203,4 +201,4 @@ loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: index_var: __dw_dbc_index - label: "{{ item.0.name }}" + label: "{{ item.0.name }}" \ No newline at end of file From 7e8855f089436c096eb123f61b93beda18ef6ad2 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 13:42:36 -0400 Subject: [PATCH 07/13] Address review comments Signed-off-by: Saravanan Raju --- roles/runtime/defaults/main.yml | 3 ++ roles/runtime/tasks/initialize_base.yml | 30 ++++++++++---------- roles/runtime/tasks/initialize_setup_aws.yml | 11 ++++--- roles/runtime/tasks/setup_aws.yml | 23 +++++++++++---- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index ddbab2bf..90bc67ba 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -72,6 +72,9 @@ run__dw_overlay_network: "{{ dw.overlay_network | bool }}" run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" +run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" +run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" +run__dw_default_dbc_suffix: "{{ dw.default_dbc_suffix | default('-dl-default') }}" run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 41c42a87..6dca5c64 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -170,11 +170,10 @@ run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}" vars: include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}" - use_default_dbc: "{{ True if __dw_config.name is not defined else False | bool }}" config: name: "{{ __dw_config.name | default('') }}" load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}" - use_default_dbc: "{{ use_default_dbc }}" + use_default_dbc: "{{ __dw_config.name is undefined | bool }}" virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}" loop: "{{ run__dw_definitions }}" loop_control: @@ -186,19 +185,20 @@ run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}" vars: config: - name: "{{ item.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" - dbc_name: "{{ item.0.name }}" - use_default_dbc: "{{ item.0.use_default_dbc }}" - type: "{{ item.1.type | default('hive') }}" - template: "{{ item.1.template | default('xsmall') }}" - autoscaling_min_nodes: "{{ item.1.autoscaling.min_nodes | default(None) }}" - autoscaling_max_nodes: "{{ item.1.autoscaling.max_nodes | default(None) }}" - common_configs: "{{ item.1.configs.common_configs | default({}) }}" - application_configs: "{{ item.1.configs.application_configs | default({}) }}" - ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}" - enable_sso: "{{ item.1.configs.enable_sso | default(None) }}" - tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}" + name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}" + dbc_name: "{{ __dw_config.0.name }}" + use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" + type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" + template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" + autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}" + autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}" + common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}" + application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}" + ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}" + enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}" + tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: + loop_var: __dw_config index_var: __dw_dbc_index - label: "{{ item.0.name }}" \ No newline at end of file + label: "{{ __dw_config.0.name }}" \ No newline at end of file diff --git a/roles/runtime/tasks/initialize_setup_aws.yml b/roles/runtime/tasks/initialize_setup_aws.yml index 5473311a..d5519c8a 100644 --- a/roles/runtime/tasks/initialize_setup_aws.yml +++ b/roles/runtime/tasks/initialize_setup_aws.yml @@ -15,7 +15,7 @@ # limitations under the License. - name: Discover AWS Public and Private VPC Subnets - when: not infra__aws_subnet_ids + when: infra__aws_subnet_ids is undefined block: - name: Query AWS Public Subnets amazon.aws.ec2_vpc_subnet_info: @@ -49,18 +49,17 @@ label: "{{ __aws_private_subnet_item.subnet_id }}" loop: "{{ __aws_private_subnets_info.subnets }}" - - name: Assign AWS Private Subnet IDs in not assinged - when: run__datahub_private_subnet_ids is undefined + - name: Set default AWS Private Subnet if there are no Private Subnets ansible.builtin.set_fact: - run__datahub_private_subnet_ids: "{{ [] }}" + run__datahub_private_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) }}" - name: Set fact for AWS Subnet IDs when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined ansible.builtin.set_fact: - run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}" + run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | union(run__datahub_public_subnet_ids) }}" - name: Set fact for AWS Subnet IDs by assignment - when: infra__aws_subnet_ids + when: infra__aws_subnet_ids is defined block: - name: Set fact for All AWS Subnet IDs by assignment ansible.builtin.set_fact: diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 8ed1edd3..ccd17f11 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -67,6 +67,19 @@ env: "{{ run__env_name }}" register: run__dw_list + - name: Fetch all the Database Catalogs under the cluster + cloudera.cloud.dw_database_catalog_info: + cluster_id: "{{ run__dw_list.clusters[0].id }}" + register: __dbc_list + + - name: Set default Database catalog id for the cluster + when: __dw_dbc.name | regex_search('.*'+run__dw_default_dbc_suffix+'$') + ansible.builtin.set_fact: + __default_dbc_id: "{{ __dw_dbc.id }}" + loop: "{{ __dbc_list.database_catalogs }}" + loop_control: + loop_var: __dw_dbc + - name: Create CDP DW Database catalogs when: not __dw_dbc_config.use_default_dbc cloudera.cloud.dw_database_catalog: @@ -97,9 +110,9 @@ label: "{{ __dw_dbc_build.__dw_dbc_config.name }}" - name: Set CDP DW Database catalog name to id map - when: __dw_dbc_build_async.database_catalogs is defined + when: __dw_dbc_build_async.database_catalog is defined ansible.builtin.set_fact: - run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalogs[0].name : __dw_dbc_build_async.database_catalogs[0].id}) }}" + run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.database_catalog.name : __dw_dbc_build_async.database_catalog.id}) }}" loop: "{{ __dw_dbc_builds_async.results }}" loop_control: loop_var: __dw_dbc_build_async @@ -107,12 +120,12 @@ - name: Create CDP DW Virtual warehouse cloudera.cloud.dw_virtual_warehouse: cluster_id: "{{ run__dw_list.clusters[0].id }}" - dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}" + dbc_id: "{{ __dw_vw_config.use_default_dbc | ternary(__default_dbc_id, run__dw_dbc_ids[__dw_vw_config.dbc_name]) }}" type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes | int }}" - autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes | int }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}" common_configs: "{{ __dw_vw_config.common_configs }}" application_configs: "{{ __dw_vw_config.application_configs }}" ldap_groups: "{{ __dw_vw_config.ldap_groups }}" From 09c38ce0a4c17453b2c4ff894727d6881f41639a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 15:38:06 -0400 Subject: [PATCH 08/13] Fix missing defaults for DW overlay network and private worker nodes Signed-off-by: Webster Mudge --- roles/runtime/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 90bc67ba..bbe40419 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -68,9 +68,9 @@ run__dw_definitions: "{{ dw.definitions | default([{}]) }}" run__dw_dbc_suffix: "{{ dw.dbc.suffix | default('dbc') }}" run__dw_vw_suffix: "{{ dw.vw.suffix | default('vw') }}" run__dw_tags: "{{ dw.tags | default(common__tags) }}" -run__dw_overlay_network: "{{ dw.overlay_network | bool }}" +run__dw_overlay_network: "{{ dw.overlay_network | default(False) | bool }}" run__dw_private_load_balancer: "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}" -run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | bool }}" +run__dw_private_worker_nodes: "{{ dw.private_worker_nodes | default(False) | bool }}" run__dw_force_delete: "{{ dw.force_delete | default (run__force_teardown) }}" run__dw_default_vw_type: "{{ dw.default_vw_type | default('hive') }}" run__dw_default_template_type: "{{ dw.default_template_type | default('xsmall') }}" From 56c65177b4510bfa8320c47eab7d52346cd9cbd6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 15:38:46 -0400 Subject: [PATCH 09/13] Add DW configuration parameters to documentation Signed-off-by: Webster Mudge --- docs/configuration.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/configuration.yml b/docs/configuration.yml index 264a848d..5112a663 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -25,7 +25,33 @@ df: persist: dw: definitions: + - name: + use_default_dbc: + load_demo_data: + virtual_warehouses: + - name: + vw_type: + template: + autoscaling: + min_cluster: + max_cluster: + tags: + configs: + common_configs: + application_configs: + enable_sso: + ldap_groups: suffix: + vw: + suffix: + tags: + overlay_network: + private_load_balancer: + private_worker_nodes: + force_delete: + default_vw_type: + default_template_type: + default_dbc_suffix: env: aws: policy: From 52f454cfe20214551e7500d61ca62e9805f85a1b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 16:46:30 -0400 Subject: [PATCH 10/13] Fix configuration parameter names Signed-off-by: Webster Mudge --- docs/configuration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.yml b/docs/configuration.yml index 5112a663..ba824f30 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -30,11 +30,11 @@ dw: load_demo_data: virtual_warehouses: - name: - vw_type: + type: template: autoscaling: - min_cluster: - max_cluster: + min_nodes: + max_nodes: tags: configs: common_configs: From 43754aa9dee2566466060a19c9a7c718c189045a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 16:47:11 -0400 Subject: [PATCH 11/13] Refactor to use 'omit' for DW config and setup Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 6dca5c64..940550fd 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -190,12 +190,12 @@ use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" - autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(None) }}" - autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(None) }}" - common_configs: "{{ __dw_config.1.configs.common_configs | default({}) }}" - application_configs: "{{ __dw_config.1.configs.application_configs | default({}) }}" - ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(None) }}" - enable_sso: "{{ __dw_config.1.configs.enable_sso | default(None) }}" + autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(omit) }}" + autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(omit) }}" + common_configs: "{{ __dw_config.1.configs.common_configs | default(omit) }}" + application_configs: "{{ __dw_config.1.configs.application_configs | default(omit) }}" + ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(omit) }}" + enable_sso: "{{ __dw_config.1.configs.enable_sso | default(omit) }}" tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: From 9a4567a96756f14169551fd46a784c4f926314fd Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 16:47:33 -0400 Subject: [PATCH 12/13] Remove extraneous DW cluster setup wait Signed-off-by: Webster Mudge --- roles/runtime/tasks/setup_aws.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index ccd17f11..70359b21 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -51,12 +51,9 @@ register: __dw_builds - name: Wait for CDP DW cluster setup to complete + when: false ansible.builtin.async_status: jid: "{{ __dw_builds.ansible_job_id }}" - #loop_control: - # loop_var: __opdb_build - # label: "{{ __opdb_build.__opdb_config.name }}" - #loop: "{{ __opdb_builds.results }}" register: __dw_builds_async until: __dw_builds_async.finished retries: 120 From 42f33cb3ef4d30201c19a2bd0a0f3e8d099cdc55 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 17:12:13 -0400 Subject: [PATCH 13/13] Refactor DW VM module defaults and omit logic Signed-off-by: Webster Mudge --- roles/runtime/tasks/initialize_base.yml | 8 ++------ roles/runtime/tasks/setup_aws.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/roles/runtime/tasks/initialize_base.yml b/roles/runtime/tasks/initialize_base.yml index 940550fd..e17ee5f2 100644 --- a/roles/runtime/tasks/initialize_base.yml +++ b/roles/runtime/tasks/initialize_base.yml @@ -190,13 +190,9 @@ use_default_dbc: "{{ __dw_config.0.use_default_dbc }}" type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}" template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}" - autoscaling_min_nodes: "{{ __dw_config.1.autoscaling.min_nodes | default(omit) }}" - autoscaling_max_nodes: "{{ __dw_config.1.autoscaling.max_nodes | default(omit) }}" - common_configs: "{{ __dw_config.1.configs.common_configs | default(omit) }}" - application_configs: "{{ __dw_config.1.configs.application_configs | default(omit) }}" - ldap_groups: "{{ __dw_config.1.configs.ldap_groups | default(omit) }}" - enable_sso: "{{ __dw_config.1.configs.enable_sso | default(omit) }}" tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}" + autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}" + configs: "{{ __dw_config.1.configs | default({}) }}" loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}" loop_control: loop_var: __dw_config diff --git a/roles/runtime/tasks/setup_aws.yml b/roles/runtime/tasks/setup_aws.yml index 70359b21..cebf967b 100644 --- a/roles/runtime/tasks/setup_aws.yml +++ b/roles/runtime/tasks/setup_aws.yml @@ -121,13 +121,13 @@ type: "{{ __dw_vw_config.type }}" name: "{{ __dw_vw_config.name }}" template: "{{ __dw_vw_config.template }}" - autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling_min_nodes }}" - autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling_max_nodes }}" - common_configs: "{{ __dw_vw_config.common_configs }}" - application_configs: "{{ __dw_vw_config.application_configs }}" - ldap_groups: "{{ __dw_vw_config.ldap_groups }}" - enable_sso: "{{ __dw_vw_config.enable_sso | bool }}" tags: "{{ __dw_vw_config.tags }}" + autoscaling_min_nodes: "{{ __dw_vw_config.autoscaling.min_nodes | default(omit) }}" + autoscaling_max_nodes: "{{ __dw_vw_config.autoscaling.max_nodes | default(omit) }}" + common_configs: "{{ __dw_vw_config.configs.common_configs | default(omit) }}" + application_configs: "{{ __dw_vw_config.configs.application_configs | default(omit) }}" + ldap_groups: "{{ __dw_vw_config.configs.ldap_groups | default(omit) }}" + enable_sso: "{{ __dw_vw_config.configs.enable_sso | default(omit) }}" wait: yes async: 3600 # 1 hour timeout poll: 0