Support CDW VW changes

Author: raju-saravanan

roles/common/defaults/main.yml

@@ -121,7 +121,7 @@ common__env_name_suffix:                  "{{ env.suffix | default(common__env_s
 
 common__datalake_name:                    "{{ env.datalake.name | default([common__namespace_cdp, common__datalake_name_suffix] | join('-')) }}"
 common__datalake_name_suffix:             "{{ env.datalake.suffix | default(common__datalake_suffix) }}"
-common__tunnel:                           "{{ env.tunnel | default(False) }}"
+common__tunnel:                           "{{ env.tunnel | default(True) }}"
 common__public_endpoint_access:           "{{ env.public_endpoint_access | default(not common__tunnel) }}"
 
 common__env_admin_password:                     "{{ globals.admin_password | mandatory }}"

roles/runtime/defaults/main.yml

@@ -59,7 +59,12 @@ run__ml_tags:                       "{{ ml.tags | default(common__tags) }}"
 run__ml_public_loadbalancer:        "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}"
 
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
-run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
+run__dw_dbc_suffix:                 "{{ dw.dbc.suffix | default('dbc') }}"
+run__dw_vw_suffix:                  "{{ dw.vw.suffix | default('vw') }}"
+run__dw_tags:                       "{{ dw.tags | default(common__tags) }}"
+run__dw_overlay_network:            "{{ dw.overlay_network | bool }}"
+run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
+run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | bool }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"

roles/runtime/tasks/initialize_base.yml

@@ -188,7 +188,42 @@
 - name: Prepare for CDP DW experiences
   when: run__include_dw
   block:
-    - name: Retrieve CDP DW experiences
-      cloudera.cloud.dw_cluster_info:
-        env: "{{ run__env_name }}"
-      register: run__dw_list
+    - name: Construct CDP DW Data catalog configurations
+      ansible.builtin.set_fact:
+        run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        use_default_dbc: "{{ __dw_config.use_default_dbc | default(False) | bool }}"
+        default_dbc_name: "{{ None if use_default_dbc else [run__namespace, run__dw_dbc_suffix, __dw_config_index] | join('-') }}"
+        config:
+          name: "{{ __dw_config.name | default(default_dbc_name) }}"
+          load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
+          use_default_dbc: "{{ use_default_dbc }}"
+          virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
+      loop: "{{ run__dw_definitions }}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_config_index
+
+    - name: Construct CDP DW Virtual warehouse configurations
+      ansible.builtin.set_fact:
+        run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
+      vars:
+        dbc_name: "{{ item.0.name }}"
+        use_default_dbc: "{{ item.0.use_default_dbc }}"
+        config:
+          name: "{{ item.1.name | default([dbc_name, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          dbc_name: "{{ dbc_name }}"
+          use_default_dbc: "{{ use_default_dbc }}"
+          vw_type: "{{ item.1.vw_type | default('hive') }}"
+          template: "{{ item.1.template | default('xsmall') }}"
+          autoscaling_min_cluster: "{{ item.1.autoscaling.min_cluster | default(None) }}"
+          autoscaling_max_cluster: "{{ item.1.autoscaling.max_cluster | default(None) }}"
+          common_configs: "{{ item.1.configs.common_configs | default({}) }}"
+          application_configs: "{{ item.1.configs.application_configs | default({}) }}"
+          ldap_groups: "{{ item.1.configs.ldap_groups | default(None) }}"
+          enable_sso: "{{ item.1.configs.enable_sso | default(None) }}"
+          tags: "{{ item.1.tags | default({}) | combine(run__dw_tags) }}"
+      loop: "{{ run__dw_dbc_configs | subelements('virtual_warehouses')}}"
+      loop_control:
+        index_var: __dw_dbc_index

roles/runtime/tasks/initialize_setup_aws.yml

@@ -49,10 +49,15 @@
         label: "{{ __aws_private_subnet_item.subnet_id }}"
       loop: "{{ __aws_private_subnets_info.subnets }}"
 
+    - name: Assign AWS Private Subnet IDs in not assinged
+      when: run__datahub_private_subnet_ids is undefined
+      ansible.builtin.set_fact:
+        run__datahub_private_subnet_ids: "{{ [] }}"
+
     - name: Set fact for AWS Subnet IDs
       when: __aws_public_subnets_info is defined or __aws_private_subnets_info is defined
       ansible.builtin.set_fact:
-        run__datahub_subnet_ids: "{{ run__datahub_public_subnet_ids | default([]) | union(run__datahub_private_subnet_ids) }}"
+        run__datahub_subnet_ids: "{{ run__datahub_private_subnet_ids | default([]) | union(run__datahub_public_subnet_ids) }}"
 
 - name: Set fact for AWS Subnet IDs by assignment
   when: infra__aws_subnet_ids is defined

roles/runtime/tasks/setup_aws.yml

@@ -37,27 +37,104 @@
 - name: Setup CDP DW cluster on AWS
   when: run__include_dw
   block:
-    - name: Execute CDP DW cluster setup
-      cloudera.cloud.dw_cluster:
+#    - name: Execute CDP DW cluster setup
+#      cloudera.cloud.dw_cluster:
+#        env: "{{ run__env_name }}"
+#        overlay: "{{ run__dw_overlay_network }}"
+#        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
+#        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+#        state: present
+#        wait: yes
+#      async: 3600 # 1 hour timeout
+#      poll: 0
+#      register: __dw_builds
+#
+#    - name: Wait for CDP DW cluster setup to complete
+#      ansible.builtin.async_status:
+#        jid: "{{ __dw_builds.ansible_job_id }}"
+#      #loop_control:
+#      #  loop_var: __opdb_build
+#      #  label: "{{ __opdb_build.__opdb_config.name }}"
+#      #loop: "{{ __opdb_builds.results }}"
+#      register: __dw_builds_async
+#      until: __dw_builds_async.finished
+#      retries: 120
+#      delay: 30
+
+    - name: Retrieve CDP DW experiences
+      cloudera.cloud.dw_cluster_info:
         env: "{{ run__env_name }}"
-        overlay: no
-        # TODO - Allow direct assignment (will need to coordinate with infra role)
-        aws_public_subnets: "{{ run__datahub_public_subnet_ids }}"
-        aws_private_subnets: "{{ run__datahub_private_subnet_ids }}"
+      register: run__dw_list
+
+    - name: Create CDP DW Database catalogs
+      when: not __dw_dbc_config.use_default_dbc
+      cloudera.cloud.dw_dbc:
+        cluster_id : "{{ run__dw_list.clusters[0].id }}"
+        name: "{{ __dw_dbc_config.name }}"
+        load_demo_data: "{{ __dw_dbc_config.load_demo_data }}"
         state: present
         wait: yes
       async: 3600 # 1 hour timeout
       poll: 0
-      register: __dw_builds
+      loop: "{{ run__dw_dbc_configs }}"
+      loop_control:
+        loop_var: __dw_dbc_config
+      register: __dw_dbc_builds
 
-    - name: Wait for CDP DW cluster setup to complete
+    - name: Wait for CDP DW Database catalogs setup to complete
+      when: __dw_dbc_build.ansible_job_id is defined
       ansible.builtin.async_status:
-        jid: "{{ __dw_builds.ansible_job_id }}"
-      #loop_control:
-      #  loop_var: __opdb_build
-      #  label: "{{ __opdb_build.__opdb_config.name }}"
-      #loop: "{{ __opdb_builds.results }}"
-      register: __dw_builds_async
-      until: __dw_builds_async.finished
+        jid: "{{ __dw_dbc_build.ansible_job_id }}"
+      register: __dw_dbc_builds_async
+      until: __dw_dbc_builds_async.finished
       retries: 120
-      delay: 30
+      delay: 30
+      loop: "{{ __dw_dbc_builds.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build
+
+    - name: Set CDP DW Database catalog name to id map
+      when: __dw_dbc_build_async.dbcs is defined
+      ansible.builtin.set_fact:
+        run__dw_dbc_ids: "{{ run__dw_dbc_ids | default({}) | combine({ __dw_dbc_build_async.dbcs[0].name : __dw_dbc_build_async.dbcs[0].id}) }}"
+      loop: "{{ __dw_dbc_builds_async.results }}"
+      loop_control:
+        loop_var: __dw_dbc_build_async
+
+    - name: Create CDP DW Virtual warehouse
+      cloudera.cloud.dw_vw:
+        cluster_id: "{{ run__dw_list.clusters[0].id }}"
+        dbc_id: "{{ run__dw_dbc_ids[__dw_vw_config.dbc_name] if not __dw_vw_config.use_default_dbc else run__dw_list.clusters[0].dbcs[0].id}}"
+        vw_type: "{{ __dw_vw_config.vw_type }}"
+        name: "{{ __dw_vw_config.name }}"
+        template: "{{ __dw_vw_config.template }}"
+        autoscaling_min_cluster: "{{ __dw_vw_config.autoscaling_min_cluster | int }}"
+        autoscaling_max_cluster: "{{ __dw_vw_config.autoscaling_max_cluster | int }}"
+        common_configs: "{{ __dw_vw_config.common_configs }}"
+        application_configs: "{{ __dw_vw_config.application_configs }}"
+        ldap_groups: "{{ __dw_vw_config.ldap_groups }}"
+        enable_sso: "{{ __dw_vw_config.enable_sso | bool }}"
+        tags: "{{ __dw_vw_config.tags }}"
+        wait: yes
+      async: 3600 # 1 hour timeout
+      poll: 0
+      register: __dw_vw_builds
+      loop: "{{ run__dw_vw_configs }}"
+      loop_control:
+        loop_var: __dw_vw_config
+
+    - name: Wait for CDP DW Virtual warehouse setup to complete
+      when: __dw_vw_build.ansible_job_id is defined
+      ansible.builtin.async_status:
+        jid: "{{ __dw_vw_build.ansible_job_id }}"
+      register: __dw_vw_builds_async
+      until: __dw_vw_builds_async.finished
+      retries: 120
+      delay: 30
+      loop: "{{ __dw_vw_builds.results }}"
+      loop_control:
+        loop_var: __dw_vw_build
+
+    - name: Print Virtual Warehouses
+      ansible.builtin.debug:
+        msg: "Database Cluster : {{ __dw_vw_builds_async }}"