Support for CDW Data Service

docs/configuration.yml

@@ -14,6 +14,7 @@ datahub:
     gcp:
   suffix:
   tags:
+  force_delete:
 de:
   definitions:
   suffix:
@@ -32,10 +33,37 @@ df:
   teardown:
     persist:
   force_delete:
-  terminate_deployments:
 dw:
   definitions:
+    - name:
+      use_default_dbc:
+      load_demo_data:
+      virtual_warehouses:
+        - name:
+          type:
+          template:
+          autoscaling:
+            min_nodes:
+            max_nodes:
+          tags:
+          configs:
+            common_configs:
+            application_configs:
+            enable_sso:
+            ldap_groups:
   suffix:
+  vw:
+    suffix:
+    type:
+    template:
+  dbc:
+    suffix:
+    default_suffix: 
+  tags:
+  overlay_network:
+  private_load_balancer:
+  private_worker_nodes:
+  force_delete:
 env:
   aws:
     policy:
@@ -195,7 +223,7 @@ globals:
   dynamic_inventory:
     vm:
       count:
-      os:
+  force_teardown:
   gcloud_credential_file:
   infra_type:
   labels:
@@ -353,6 +381,7 @@ ml:
   suffix:
   tags:
   public_loadbalancer:
+  force_delete:
 opdb:
   definitions:
   suffix:

roles/common/defaults/main.yml

@@ -93,11 +93,17 @@ common__aws_private_subnet_ids:           "{{ infra.aws.vpc.existing.private_sub
 common__aws_region:                       "{{ infra.aws.region | default('eu-west-1') }}"
 common__aws_profile:                      "{{ infra.aws.profile | default('') }}"
 common__aws_role_suffix:                  "{{ infra.aws.role.suffix | default(common__role_suffix) }}"
+
 common__aws_datalake_admin_role_name:     "{{ env.aws.role.name.datalake_admin | default([common__namespace, common__aws_datalake_admin_suffix, common__aws_role_suffix] | join('-')) }}"
 common__aws_datalake_admin_suffix:        "{{ env.aws.role.label.datalake_admin | default(common__datalake_admin_suffix) }}"
 common__aws_idbroker_role_name:           "{{ env.aws.role.name.idbroker | default([common__namespace, common__aws_idbroker_suffix, common__aws_role_suffix] | join('-')) }}"
 common__aws_idbroker_suffix:              "{{ env.aws.role.label.idbroker | default(common__idbroker_suffix) }}"
 
+
+common__aws_vpc_id:                       "{{ infra.aws.vpc.existing.vpc_id | default('') }}"
+common__aws_public_subnet_ids:            "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}"
+common__aws_private_subnet_ids:           "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
+
 # Azure Infra
 common__azure_storage_name:               "{{ infra.azure.storage.name | default(common__storage_name | replace('-','')) }}"
 

roles/common/meta/main.yml

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 galaxy_info:
+  role_name: platform
+  namespace: cloudera
   author: Webster Mudge ([email protected])
   description: >
     Shared configuration variables managed by role dependency. 

roles/infrastructure/defaults/main.yml

@@ -83,9 +83,9 @@ infra__vpc_user_ports:              "{{ infra.vpc.user_ports | default([infra__a
 infra__vpc_user_cidr:               "{{ infra.vpc.user_cidr | default([]) }}"
 infra__vpc_tunneled_cidr:           "{{ infra.vpc.tunneled_cidr | default([]) }}"
 
-infra__aws_vpc_id:                  "{{ infra.aws.vpc.existing.vpc_id | default('') }}"
-infra__aws_public_subnet_ids:       "{{ infra.aws.vpc.existing.public_subnet_ids | default([]) }}"
-infra__aws_private_subnet_ids:      "{{ infra.aws.vpc.existing.private_subnet_ids | default([]) }}"
+infra__aws_vpc_id:                  "{{ common__aws_vpc_id }}"
+infra__aws_public_subnet_ids:       "{{ common__aws_public_subnet_ids }}"
+infra__aws_private_subnet_ids:      "{{ common__aws_private_subnet_ids }}"
 
 infra__security_group_knox_name:    "{{ common__security_group_knox_name }}"
 infra__security_group_default_name: "{{ common__security_group_default_name }}"

roles/infrastructure/tasks/initialize_aws.yml

@@ -115,7 +115,6 @@
 
         - name: Set facts for existing AWS Public Subnet IDs
           ansible.builtin.set_fact:
-            infra__aws_public_subnet_ids: "{{ infra__aws_public_subnet_ids }}"
             infra__aws_subnet_ids: "{{ infra__aws_subnet_ids | default([]) | union(infra__aws_public_subnet_ids) }}"
             infra__aws_vpc_id: "{{ __aws_public_subnets_info.subnets | map(attribute='vpc_id') | list | first }}"
 

roles/platform/defaults/main.yml

@@ -103,6 +103,9 @@ plat__aws_storage_suffix:                     "{{ env.aws.storage.suffix | defau
 plat__aws_role_tags:                          "{{ env.aws.role.tags | default({}) }}"
 plat__aws_policy_tags:                        "{{ env.aws.policy.tags | default({}) }}"
 plat__aws_storage_tags:                       "{{ env.aws.storage.tags | default({}) }}"
+plat__aws_vpc_id:                             "{{ common__aws_vpc_id }}"
+plat__aws_public_subnet_ids:                  "{{ common__aws_public_subnet_ids }}"
+plat__aws_private_subnet_ids:                 "{{ common__aws_private_subnet_ids }}"
 
 plat__aws_xaccount_suffix:                    "{{ env.aws.role.label.cross_account | default(common__xaccount_suffix) }}"
 plat__aws_idbroker_suffix:                    "{{ common__aws_idbroker_suffix }}"

roles/platform/meta/main.yml

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 galaxy_info:
+  role_name: platform
+  namespace: cloudera
   author: Webster Mudge ([email protected])
   description: >
     Deployment and management of Cloudera Data Platform (CDP) Public Cloud core

roles/platform/tasks/initialize_aws.yml

@@ -17,4 +17,9 @@
 - name: Retrieve AWS Caller details
   amazon.aws.aws_caller_info:
   register: __aws_caller_info
-  failed_when: __aws_caller_info.account is not defined
+  failed_when: __aws_caller_info.account is not defined
+
+- name: Confirm AWS SSH Public Key ID exists
+  ansible.builtin.command: aws ec2 describe-key-pairs --region "{{ plat__region }}" --key-name "{{ plat__public_key_id }}"
+  register: __aws_ssh_key_pair
+  failed_when: __aws_ssh_key_pair.rc != 0

roles/platform/tasks/initialize_setup_aws.yml

@@ -48,7 +48,7 @@
     plat__aws_xaccount_account_id: "{{ plat__cdp_xaccount_account_id }}"
 
 # Runlevel first, upstream second, and discover third
-- name: Discover AWS VPC if not defined
+- name: Discover AWS VPC if not defined or established by Infrastructure
   when: plat__aws_vpc_id == "" and infra__aws_vpc_id is undefined
   block:
     - name: Query AWS VPC by name
@@ -70,13 +70,12 @@
         plat__aws_vpc_id: "{{ __aws_vpc_info.vpcs[0].id }}"
 
 - name: Set fact for AWS VPC ID if established by Infrastructure
-  when: infra__aws_vpc_id is defined
+  when: plat__aws_vpc_id == "" and infra__aws_vpc_id is defined
   ansible.builtin.set_fact:
     plat__aws_vpc_id: "{{ infra__aws_vpc_id }}"
 
-# Runlevel first, upstream second, and discover third
-- name: Handle AWS Public and Private VPC Subnets if not defined
-  when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids
+- name: Handle AWS Subnet IDs if not defined
+  when: not plat__aws_public_subnet_ids or not plat__aws_private_subnet_ids # Defaults are empty lists
   block:
     - name: Query AWS Subnets
       amazon.aws.ec2_vpc_subnet_info:
@@ -143,13 +142,15 @@
   ansible.builtin.set_fact:
     plat__endpoint_access_scheme: "PUBLIC"
 
+# TODO Collapse the two SG queries together
 - name: Discover AWS Security Group for Knox
   when: infra__aws_security_group_knox_id is undefined
   block:
     - name: Query AWS Security Group for Knox
       amazon.aws.ec2_group_info:
         region: "{{ plat__region }}"
         filters:
+          vpc-id: "{{ plat__aws_vpc_id }}"
           group-name: "{{ plat__security_group_knox_name }}"
       register: __aws_security_group_knox_info
 
@@ -170,6 +171,7 @@
       amazon.aws.ec2_group_info:
         region: "{{ plat__region }}"
         filters:
+          vpc-id: "{{ plat__aws_vpc_id }}"
           group-name: "{{ plat__security_group_default_name }}"
       register: __aws_security_group_default_info
 

roles/platform/tasks/setup_aws_env.yml

@@ -31,7 +31,7 @@
     subnet_ids: "{{ plat__aws_public_subnet_ids | union(plat__aws_private_subnet_ids) }}"
     tags: "{{ plat__tags }}"
     tunnel: "{{ plat__tunnel }}"
-    endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}"
+    endpoint_access_scheme: "{{ plat__public_endpoint_access | ternary('PUBLIC', omit) }}"
     endpoint_access_subnets: "{{ plat__aws_public_subnet_ids | default(omit) }}"
     freeipa:
       instanceCountByGroup: "{{ plat__env_freeipa }}"

roles/runtime/defaults/main.yml

@@ -71,8 +71,17 @@ run__de_force_delete:               "{{ de.force_delete | default (run__force_te
 run__de_vc_suffix:                  "{{ de.vc.suffix | default('vc') }}"
 
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
-run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
+run__dw_dbc_suffix:                 "{{ dw.dbc.suffix | default('dbc') }}"
+run__dw_vw_suffix:                  "{{ dw.vw.suffix | default('vw') }}"
+run__dw_tags:                       "{{ dw.tags | default(common__tags) }}"
+run__dw_overlay_network:            "{{ dw.overlay_network | default(False) | bool }}"
+run__dw_private_load_balancer:      "{{ dw.private_load_balancer | default(not run__public_endpoint_access) }}"
+run__dw_private_worker_nodes:       "{{ dw.private_worker_nodes | default(False) | bool }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
+run__dw_default_vw_type:            "{{ dw.default_vw.type | default('hive') }}"
+run__dw_default_template_type:      "{{ dw.default_template.type | default('xsmall') }}"
+run__dw_default_dbc_suffix:         "{{ dw.default_dbc.suffix | default('dl-default') }}"
+run__dw_default_dbc:                "{{ dw.default_dbc.name | default([run__env_name, run__dw_default_dbc_suffix] | join('-')) }}"
 
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"

roles/runtime/tasks/initialize_base.yml

@@ -52,6 +52,7 @@
       ansible.builtin.set_fact:
         run__cdp_datalake_version: "{{ __cdp_datalake_version_info.versions[0].runtimeVersion | trim }}"
 
+# TODO Discover version if upstream is not present
 - name: Set fact for CDP Datalake version by assignment
   when: plat__cdp_datalake_version is defined
   ansible.builtin.set_fact:
@@ -64,7 +65,7 @@
     - name: Retrieve Image Catalog File
       ansible.builtin.uri:
         url: "{{ run__datahub_image_catalog_url }}"
-      #no_log: yes
+      no_log: yes
       register: __datahub_image_catalog
 
     - name: Set fact for latest CDP Image in Catalog
@@ -165,19 +166,51 @@
         loop_var: __ml_config
         label: "{{ config.name }}"
 
-- name: Prepare for CDP DE Service experiences
-  when: run__include_de
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
   block:
-    - name: Construct CDP DE Service configurations
+    - name: Construct CDP DW Data Catalog configurations
       ansible.builtin.set_fact:
-        run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}"
+        run__dw_dbc_configs: "{{ run__dw_dbc_configs | default([]) | union([config]) }}"
       vars:
-        include: "{{ lookup('template', __de_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
-        overlay_tags: "{{ __de_config.tags | default(include.tags) | default(run__de_tags) }}"
+        include: "{{ lookup('template', __dw_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
         config:
-          name: "{{ __de_config.name | default([run__namespace_cdp, __de_config.suffix | default(include.suffix) | default(run__de_suffix)] | join('-')) }}"
-          raw: "{{ __de_config }}"
-      loop: "{{ run__de_definitions }}"
+          name: "{{ __dw_config.name | default(run__dw_default_dbc) }}"
+          load_demo_data: "{{ __dw_config.load_demo_data | default(False) | bool }}"
+          virtual_warehouses: "{{ __dw_config.virtual_warehouses | default([]) }}"
+      loop: "{{ run__dw_definitions }}"
       loop_control:
-        loop_var: __de_config
+        loop_var: __dw_config
+        index_var: __dw_config_index
+
+    - name: Construct CDP DW Virtual Warehouse configurations
+      ansible.builtin.set_fact:
+        run__dw_vw_configs: "{{ run__dw_vw_configs | default([]) | union([config]) }}"
+      vars:
+        config:
+          dbc_name: "{{ __dw_config.0.name }}"
+          name: "{{ __dw_config.1.name | default([run__namespace, run__dw_vw_suffix ,__dw_dbc_index] | join('-')) }}"
+          type: "{{ __dw_config.1.type | default(run__dw_default_vw_type) }}"
+          template: "{{ __dw_config.1.template | default(run__dw_default_template_type) }}"
+          tags: "{{ __dw_config.1.tags | default({}) | combine(run__dw_tags) }}"
+          autoscaling: "{{ __dw_config.1.autoscaling | default({}) }}"
+          configs: "{{ __dw_config.1.configs | default({}) }}"
+      loop: "{{ run__dw_dbc_configs | default({}) | subelements('virtual_warehouses')}}"
+      loop_control:
+        loop_var: __dw_config
+        index_var: __dw_dbc_index
         label: "{{ config.name }}"
+
+    - name: Check CDP DW Virtual Warehouse tags
+      ansible.builtin.assert:
+        that:
+          - __dw_vw_config.tags | dict2items | rejectattr('value', 'regex', '[^-_a-zA-Z0-9.=:+@]+') | list
+        fail_msg: 
+          - "A tag in Data Warehouse, '{{ __dw_vw_config.name }}', does not meet requirements;"
+          - "current tags: {{ __dw_vw_config.tags}}."
+          - "Allowed characters in tags are letters, numbers and the following characters: _.:/=+-@"
+        quiet: yes
+      loop_control:
+        loop_var: __dw_vw_config
+        label: "{{ __dw_vw_config.name }}"
+      loop: "{{ run__dw_vw_configs }}"

roles/runtime/tasks/initialize_setup.yml

@@ -29,3 +29,19 @@
     - dw
     - opdb
     - dh
+
+- name: Prepare for CDP DW experiences
+  when: run__include_dw
+  block:
+    - name: Confirm public subnet count CDP DW public load balancer
+      when: not run__dw_private_load_balancer
+      ansible.builtin.assert:
+        that:
+          - run__public_subnet_ids | length == 3
+        fail_msg: "Must have exactly 3 public subnets when deploying CDP Data Warehouse with a public load balancer"
+        quiet: yes
+      tags:
+        - ml
+        - dw
+        - opdb
+        - dh

roles/runtime/tasks/initialize_teardown.yml

@@ -24,11 +24,37 @@
   when: not run__force_teardown
   ansible.builtin.include_tasks: "initialize_base.yml"
 
-- name: Discover CDP DF Deployments
-  register: run__df_service_info
+- name: Discover CDP Dataflow deployments
   when: run__include_df
   cloudera.cloud.df_info:
     name: "{{ run__env_name }}"
+  register: run__df_env_info
+
+- name: Discover CDP Data Warehouse deployments
+  when: 
+    - run__include_dw
+    - not run__force_teardown | bool or not run__dw_force_delete | bool
+  block:
+    - name: Discover CDP Data Warehouse cluster
+      cloudera.cloud.dw_cluster_info:
+        env: "{{ run__env_name }}"
+      register: __dw_list
+
+    - name: Initialize CDP Data Warehouse cluster id
+      ansible.builtin.set_fact:
+        __dw_cluster_id: "{{ __dw_list.clusters | map(attribute='id') | first | default(omit) }}"
+
+    - name: Discover CDP Data Warehouse database catalogs
+      when: __dw_cluster_id is defined
+      cloudera.cloud.dw_database_catalog_info:
+        cluster_id: "{{ __dw_cluster_id }}"
+      register: __dw_dbc_list
+
+    - name: Discover CDP Data Warehouse virtual warehouses
+      when: __dw_cluster_id is defined
+      cloudera.cloud.dw_virtual_warehouse_info:
+        cluster_id: "{{ __dw_cluster_id }}"
+      register: __dw_vw_list
 
 - name: Initialize Purge of all Runtimes in Environment
   when: