From 69ca9dfa3f129cdddbd52100863e483574e615f4 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:38 -0400 Subject: [PATCH 01/23] Support CDW DBC and VW creation Signed-off-by: Saravanan Raju --- plugins/modules/dw_cluster.py | 20 +- plugins/modules/dw_dbc.py | 178 +++++++++++++++++ plugins/modules/dw_vw.py | 353 ++++++++++++++++++++++++++++++++++ 3 files changed, 547 insertions(+), 4 deletions(-) create mode 100644 plugins/modules/dw_dbc.py create mode 100644 plugins/modules/dw_vw.py diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index 2fa5f2ed..a38fa10d 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -47,6 +47,16 @@ aliases: - environment - env_crn + overlay: + description: Set it to true to save IP addresses in the VPC by using a private IP address range for Pods in the cluster. + type: bool + required: False + default: False + private_load_balancer: + description: Set up load balancer in private subnets. + type: bool + required: False + default: False aws_public_subnets: description: List of zero or more Public AWS Subnet IDs to deploy to type: list @@ -131,7 +141,7 @@ elements: complex contains: cluster: - tyoe: dict + type: dict contains: name: description: The name of the cluster. @@ -194,6 +204,7 @@ def __init__(self, module): self.name = self._get_param('name') self.env = self._get_param('env') self.overlay = self._get_param('overlay') + self.private_load_balancer = self._get_param('private_load_balancer') self.az_subnet = self._get_param('az_subnet') self.az_enable_az = self._get_param('az_enable_az') self.aws_public_subnets = self._get_param('aws_public_subnets') @@ -281,9 +292,9 @@ def process(self): self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) else: self.name = self.cdpy.dw.create_cluster( - env_crn=env_crn, overlay=self.overlay, aws_public_subnets=self.aws_public_subnets, - aws_private_subnets=self.aws_private_subnets, az_subnet=self.az_subnet, - az_enable_az=self.az_enable_az + env_crn=env_crn, overlay=self.overlay, private_load_balancer=self.private_load_balancer, + aws_public_subnets=self.aws_public_subnets, aws_private_subnets=self.aws_private_subnets, + az_subnet=self.az_subnet, az_enable_az=self.az_enable_az ) if self.wait: self.target = self.cdpy.sdk.wait_for_state( @@ -304,6 +315,7 @@ def main(): name=dict(required=False, type='str', aliases=['id']), env=dict(required=False, type='str', aliases=['environment', 'env_crn']), overlay=dict(required=False, type='bool', default=False), + private_load_balancer=dict(required=False, type='bool', default=False), az_subnet=dict(required=False, type='str', default=None), az_enable_az=dict(required=False, type='bool', default=None), aws_public_subnets=dict(required=False, type='list', default=None), diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_dbc.py new file mode 100644 index 00000000..6292becf --- /dev/null +++ b/plugins/modules/dw_dbc.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: dw_dbc +short_description: Create CDP Database Catalog +description: + - Create CDP Database Catalog +author: + - "Webster Mudge (@wmudge)" + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + cluster_id: + description: ID of cluster where Database Catalog should be created. + type: str + required: True + name: + description: Name of the Database Catalog. + type: str + required: True + load_demo_data: + description: Set this to true if you demo data should be loaded into the Database Catalog. + type: str + required: False + wait: + description: + - Flag to enable internal polling to wait for the Data Catalog to achieve the declared state. + - If set to FALSE, the module will return immediately. + type: bool + required: False + default: True + delay: + description: + - The internal polling interval (in seconds) while the module waits for the Data Catalog to achieve the declared + state. + type: int + required: False + default: 15 + aliases: + - polling_delay + timeout: + description: + - The internal polling timeout (in seconds) while the module waits for the Data Catalog to achieve the declared + state. + type: int + required: False + default: 3600 + aliases: + - polling_timeout +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Create Database Catalog +- cloudera.cloud.dw_dbc: + name: example-database-catalog + cluster_id: example-cluster-id +''' + +RETURN = r''' +--- +dbcs: + description: The information about the named Database Catalog. + type: list + returned: always + elements: complex + contains: + id: + description: The id of the Database Catalog. + returned: always + type: str + name: + description: The name of the Database Catalog. + returned: always + type: str + status: + description: The status of the Database Catalog. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DwDbc(CdpModule): + def __init__(self, module): + super(DwDbc, self).__init__(module) + + # Set variables + self.cluster_id = self._get_param('cluster_id') + self.name = self._get_param('name') + self.load_demo_data = self._get_param('load_demo_data') + self.state = self._get_param('state') + self.wait = self._get_param('wait') + self.delay = self._get_param('delay') + self.timeout = self._get_param('timeout') + + # Initialize return values + self.dbcs = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.name = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, + load_demo_data=self.load_demo_data) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']) + self.dbcs.append(self.target) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + cluster_id=dict(required=True, type='str', aliases=['cluster_id']), + name = dict(required=True, type='str', aliases=['name']), + load_demo_data=dict(required=False, type='bool', aliases=['load_demo_data']), + state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), + wait = dict(required=False, type='bool', default=True), + delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), + timeout = dict(required=False, type='int', aliases=['polling_timeout'], default=3600) + ), + supports_check_mode=True + ) + + result = DwDbc(module) + output = dict(changed=False, dbcs=result.dbcs) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py new file mode 100644 index 00000000..fef7163b --- /dev/null +++ b/plugins/modules/dw_vw.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: dw_vw +short_description: Create CDP Virtual Warehouse +description: + - Create CDP Virtual Warehouse +author: + - "Webster Mudge (@wmudge)" + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + cluster_id: + description: ID of cluster where Virtual Warehouse should be created. + type: str + required: True + dbc_id: + description: ID of Database Catalog that the Virtual Warehouse should be attached to. + type: str + required: True + vw_type: + description: Type of Virtual Warehouse to be created. + type: str + required: True + name: + description: Name of the Virtual Warehouse. + type: str + required: True + template: + description: Name of configuration template to use. + type: str + required: False + autoscaling_min_cluster: + description: Minimum number of available nodes for Virtual Warehouse autoscaling. + type: int + required: False + autoscaling_max_cluster: + description: Maximum number of available nodes for Virtual Warehouse autoscaling. + type: int + required: False + config: + description: Configuration settings for the Virtual Warehouse. + type: dict + required: False + contains: + commonConfigs: + description: Configurations that are applied to every application in the service. + type: dict + required: False + contains: + configBlocks: List of ConfigBlocks for the application. + type: list + required: False + contains: + id: + description: ID of the ConfigBlock. Unique within an ApplicationConfig. + type: str + required: False + format: + description: Format of ConfigBlock. + type: str + required: False + content: + description: Contents of a ConfigBlock. + type: obj + required: False + contains: + keyValues: + description: Key-value type configurations. + type: obj + required: False + contains: + additionalProperties: + description: Key-value type configurations. + type: str + required: False + text: + description: Text type configuration. + type: str + required: False + json: + description: JSON type configuration. + type: str + required: False + applicationConfigs: + description: Application specific configurations. + type: dict + required: False + contains: + configBlocks: List of ConfigBlocks for the application. + type: list + required: False + contains: + id: + description: ID of the ConfigBlock. Unique within an ApplicationConfig. + type: str + required: False + format: + description: Format of ConfigBlock. + type: str + required: False + content: + description: Contents of a ConfigBlock. + type: obj + required: False + contains: + keyValues: + description: Key-value type configurations. + type: obj + required: False + contains: + additionalProperties: + description: Key-value type configurations. + type: str + required: False + text: + description: Text type configuration. + type: str + required: False + json: + description: JSON type configuration. + type: str + required: False + ldapGroups: + description: LDAP Groupnames to be enabled for auth. + type: list + required: False + enableSSO: + description: Should SSO be enabled for this VW. + type: bool + required: False + tags: + description: Tags associated with the resources. + type: dict + required: False + wait: + description: + - Flag to enable internal polling to wait for the Data Warehouse Cluster to achieve the declared state. + - If set to FALSE, the module will return immediately. + type: bool + required: False + default: True + delay: + description: + - The internal polling interval (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared + state. + type: int + required: False + default: 15 + aliases: + - polling_delay + timeout: + description: + - The internal polling timeout (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared + state. + type: int + required: False + default: 3600 + aliases: + - polling_timeout +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Create Virtual Warehouse +- cloudera.cloud.dw_vw: + name: "example-virtual-warehouse" + vw_type: "hive" + template: "xsmall" + autoscaling: + min_cluster: 3 + max_cluster: 19 + tags: + tag-key: "tag-value" + configs: + enable_sso: true + ldap_groups: ['group1','group2','group3'] +''' + +RETURN = r''' +--- +vws: + description: The information about the named CDW Virtual Warehouses. + type: list + returned: always + elements: complex + contains: + vws: + type: dict + contains: + id: + description: Id of the Virtual Warehouse created. + returned: always + type: str + name: + description: Name of the Virtual Warehouse created. + returned: always + type: str + vwType: + description: Virtual Warehouse type. + returned: always + type: str + dbcId: + description: Database Catalog ID against which Virtual Warehouse was created. + returned: always + type: str + creationDate: + description: The creation time of the cluster in UTC. + returned: always + type: str + status: + description: The status of the Virtual Warehouse. + returned: always + type: str + creator: + description: The CRN of the cluster creator. + returned: always + type: dict + contains: + crn: + type: str + description: Actor CRN + email: + type: str + description: Email address for users + workloadUsername: + type: str + description: Username for users + machineUsername: + type: str + description: Username for machine users + tags: + description: Custom tags that were used to create this Virtual Warehouse. + returned: always + type: dict +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DwVw(CdpModule): + def __init__(self, module): + super(DwVw, self).__init__(module) + + # Set variables + self.cluster_id = self._get_param('cluster_id') + self.dbc_id = self._get_param('dbc_id') + self.vw_type = self._get_param('vw_type') + self.name = self._get_param('name') + self.template = self._get_param('template') + self.autoscaling_min_cluster = self._get_param('autoscaling_min_cluster') + self.autoscaling_max_cluster = self._get_param('autoscaling_max_cluster') + self.common_configs = self._get_param('common_configs') + self.application_configs = self._get_param('application_configs') + self.ldap_groups = self._get_param('ldap_groups') + self.enable_sso = self._get_param('enable_sso') + self.tags = self._get_param('tags') + self.wait = self._get_param('wait') + self.delay = self._get_param('delay') + self.timeout = self._get_param('timeout') + + # Initialize return values + self.vws = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.name = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, + template=self.template, autoscaling_min_cluster=self.autoscaling_min_cluster, + autoscaling_max_cluster=self.autoscaling_max_cluster, + common_configs=self.common_configs, application_configs=self.application_configs, + ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, tags=self.tags) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.name), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.vw_id) + self.vws.append(self.target) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + cluster_id=dict(required=True, type='str', aliases=['cluster_id']), + dbc_id=dict(required=True, type='str', aliases=['dbc_id']), + vw_type = dict(required=True, type='str', aliases=['vw_type']), + name = dict(required=True, type='str', aliases=['name']), + template=dict(required=False, type='str', aliases=['template']), + autoscaling_min_cluster=dict(required=False, type='int', aliases=['autoscaling_min_cluster']), + autoscaling_max_cluster=dict(required=False, type='int', aliases=['autoscaling_max_cluster']), + common_configs=dict(required=False, type='dict', aliases=['common_configs']), + application_configs=dict(required=False, type='dict', aliases=['application_configs']), + ldap_groups=dict(required=False, type='list', aliases=['ldap_groups']), + enable_sso=dict(required=False, type='bool', aliases=['enable_sso']), + tags=dict(required=False, type='dict', aliases=['tags']), + wait = dict(required=False, type='bool', default=True), + delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), + timeout = dict(required=False, type='int', aliases=['polling_timeout'], default=3600) + ), + supports_check_mode=True + ) + + result = DwVw(module) + output = dict(changed=False, vws=result.vws) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() From 9202f72fe866edb607b047dbc71e2dac07a098fc Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:39 -0400 Subject: [PATCH 02/23] Fix class docs Signed-off-by: Saravanan Raju --- plugins/modules/dw_dbc.py | 2 +- plugins/modules/dw_vw.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_dbc.py index 6292becf..6184200e 100644 --- a/plugins/modules/dw_dbc.py +++ b/plugins/modules/dw_dbc.py @@ -25,7 +25,7 @@ DOCUMENTATION = r''' --- module: dw_dbc -short_description: Create CDP Database Catalog +short_description: Create CDP Data Warehouse Database Catalog description: - Create CDP Database Catalog author: diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py index fef7163b..7d9be1d0 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_vw.py @@ -25,7 +25,7 @@ DOCUMENTATION = r''' --- module: dw_vw -short_description: Create CDP Virtual Warehouse +short_description: Create CDP Data Warehouse Virtual Warehouse description: - Create CDP Virtual Warehouse author: From 9a5dae29504c89df9ccfb58ee75fd8e94106006e Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:39 -0400 Subject: [PATCH 03/23] Make vw and dbc modules idempotent Signed-off-by: Saravanan Raju --- plugins/modules/dw_dbc.py | 81 ++++++++++++++++++++++++++---- plugins/modules/dw_vw.py | 103 +++++++++++++++++++++++++++++++++----- 2 files changed, 162 insertions(+), 22 deletions(-) diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_dbc.py index 6184200e..d3804bc3 100644 --- a/plugins/modules/dw_dbc.py +++ b/plugins/modules/dw_dbc.py @@ -31,6 +31,7 @@ author: - "Webster Mudge (@wmudge)" - "Dan Chaffelson (@chaffelson)" + - "Saravanan Raju (@raju-saravanan)" requirements: - cdpy options: @@ -83,6 +84,12 @@ - cloudera.cloud.dw_dbc: name: example-database-catalog cluster_id: example-cluster-id + +# Delete Database Catalog +- cloudera.cloud.dw_dbc: + name: example-database-catalog + cluster_id: example-cluster-id + state: absent ''' RETURN = r''' @@ -133,22 +140,76 @@ def __init__(self, module): # Initialize return values self.dbcs = [] + # Initialize internal values + self.target = None + # Execute logic process self.process() @CdpModule._Decorators.process_debug def process(self): - self.name = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, - load_demo_data=self.load_demo_data) - if self.wait: - self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_dbc, - params=dict(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']), - state='Running', delay=self.delay, timeout=self.timeout - ) + cluster = self.cdpy.dw.describe_cluster(cluster_id=self.cluster_id) + if cluster is None: + self.module.fail_json(msg="Couldn't retrieve cluster info for %s " % self.cluster_id) else: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']) - self.dbcs.append(self.target) + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name) + # If Database Catalog exists + if self.target is not None: + if self.state == 'absent': + if self.module.check_mode: + self.clusters.append(self.target) + else: + if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: + self.module.warn( + "DW Database Catalog not in valid state for Delete operation: %s" % self.target['status']) + else: + _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.name) + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=self.name), + field=None, delay=self.delay, timeout=self.timeout + ) + else: + self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name) + self.clusters.append(self.target) + # Drop Done + elif self.state == 'present': + # Begin Config check + self.module.warn("DW Database Catalog already present and config validation is not implemented") + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id,dbc_id=self.name), + state='Running', delay=self.delay, timeout=self.timeout + ) + self.clusters.append(self.target) + # End Config check + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) + # End handling Database Catalog exists + else: + # Begin handling Database Catalog not found + if self.state == 'absent': + self.module.warn("DW Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) + elif self.state == 'present': + if self.module.check_mode: + pass + else: + self.name = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, + load_demo_data=self.load_demo_data) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']) + self.dbcs.append(self.target) + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) def main(): diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py index 7d9be1d0..c39d3237 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_vw.py @@ -31,6 +31,7 @@ author: - "Webster Mudge (@wmudge)" - "Dan Chaffelson (@chaffelson)" + - "Saravanan Raju (@raju-saravanan)" requirements: - cdpy options: @@ -157,6 +158,14 @@ description: Tags associated with the resources. type: dict required: False + state: + description: The declarative state of the Virtual Warehouse + type: str + required: False + default: present + choices: + - present + - absent wait: description: - Flag to enable internal polling to wait for the Data Warehouse Cluster to achieve the declared state. @@ -192,6 +201,7 @@ # Create Virtual Warehouse - cloudera.cloud.dw_vw: + cluster_id: "example-cluster-id" name: "example-virtual-warehouse" vw_type: "hive" template: "xsmall" @@ -203,6 +213,12 @@ configs: enable_sso: true ldap_groups: ['group1','group2','group3'] + +# Delete Virtual Warehouse +- cloudera.cloud.dw_vw: + cluster_id: "example-cluster-id" + name: "example-virtual-warehouse" + state: absent ''' RETURN = r''' @@ -289,6 +305,7 @@ def __init__(self, module): self.application_configs = self._get_param('application_configs') self.ldap_groups = self._get_param('ldap_groups') self.enable_sso = self._get_param('enable_sso') + self.state = self._get_param('state') self.tags = self._get_param('tags') self.wait = self._get_param('wait') self.delay = self._get_param('delay') @@ -297,33 +314,94 @@ def __init__(self, module): # Initialize return values self.vws = [] + # Initialize internal values + self.target = None + # Execute logic process self.process() @CdpModule._Decorators.process_debug def process(self): - self.name = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, + cluster = self.cdpy.dw.describe_cluster(cluster_id=self.cluster_id) + if cluster is None: + self.module.fail_json(msg="Couldn't retrieve cluster info for %s " % self.cluster_id) + else: + dbc = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.dbc_id) + if dbc is None: + self.module.fail_json(msg="Couldn't retrieve dbc info for %s " % self.dbc_id) + else: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.name) + # If Virtual Warehouse exists + if self.target is not None: + if self.state == 'absent': + if self.module.check_mode: + self.clusters.append(self.target) + else: + if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: + self.module.warn( + "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target[ + 'status']) + else: + _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.name) + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.name), + field=None, delay=self.delay, timeout=self.timeout + ) + else: + self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.name) + self.clusters.append(self.target) + # Drop Done + elif self.state == 'present': + # Begin Config check + self.module.warn("DW Virtual Warehouse already present and config validation is not implemented") + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.delete_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.name), + state='Running', delay=self.delay, timeout=self.timeout + ) + self.clusters.append(self.target) + # End Config check + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) + # End handling Virtual Warehouse exists + else: + # Begin handling Virtual Warehouse not found + if self.state == 'absent': + self.module.warn( + "DW Virtual Warehouse %s already absent in Cluster %s" % (self.name, self.cluster_id)) + elif self.state == 'present': + if self.module.check_mode: + pass + else: + self.name = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, + dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, template=self.template, autoscaling_min_cluster=self.autoscaling_min_cluster, autoscaling_max_cluster=self.autoscaling_max_cluster, common_configs=self.common_configs, application_configs=self.application_configs, ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, tags=self.tags) - if self.wait: - self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_vw, - params=dict(cluster_id=self.cluster_id, vw_id=self.name), - state='Running', delay=self.delay, timeout=self.timeout - ) - else: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.vw_id) - self.vws.append(self.target) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.name), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.vw_id) + self.vws.append(self.target) + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( cluster_id=dict(required=True, type='str', aliases=['cluster_id']), - dbc_id=dict(required=True, type='str', aliases=['dbc_id']), - vw_type = dict(required=True, type='str', aliases=['vw_type']), + dbc_id=dict(required=False, type='str', aliases=['dbc_id']), + vw_type = dict(required=False, type='str', aliases=['vw_type']), name = dict(required=True, type='str', aliases=['name']), template=dict(required=False, type='str', aliases=['template']), autoscaling_min_cluster=dict(required=False, type='int', aliases=['autoscaling_min_cluster']), @@ -333,6 +411,7 @@ def main(): ldap_groups=dict(required=False, type='list', aliases=['ldap_groups']), enable_sso=dict(required=False, type='bool', aliases=['enable_sso']), tags=dict(required=False, type='dict', aliases=['tags']), + state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), wait = dict(required=False, type='bool', default=True), delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), timeout = dict(required=False, type='int', aliases=['polling_timeout'], default=3600) From 92d8568f5397f0458befb0bd825e12adf83e4e4f Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 04/23] Make the dbc and vw module idempotent Signed-off-by: Saravanan Raju --- plugins/modules/dw_dbc.py | 135 ++++++++++++++++-------------- plugins/modules/dw_vw.py | 172 ++++++++++++++++++++------------------ 2 files changed, 162 insertions(+), 145 deletions(-) diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_dbc.py index d3804bc3..41ef2555 100644 --- a/plugins/modules/dw_dbc.py +++ b/plugins/modules/dw_dbc.py @@ -35,6 +35,11 @@ requirements: - cdpy options: + id: + description: + - If an ID is provided, that Database Catalog will be deleted if C(state=absent) + type: str + required: When state is absent cluster_id: description: ID of cluster where Database Catalog should be created. type: str @@ -82,14 +87,14 @@ # Create Database Catalog - cloudera.cloud.dw_dbc: - name: example-database-catalog - cluster_id: example-cluster-id + name: "example-database-catalog-name" + cluster_id: "example-cluster-id" # Delete Database Catalog - cloudera.cloud.dw_dbc: - name: example-database-catalog - cluster_id: example-cluster-id - state: absent + id: "example-database-id" + cluster_id: "example-cluster-id" + state: "absent" ''' RETURN = r''' @@ -129,6 +134,7 @@ def __init__(self, module): super(DwDbc, self).__init__(module) # Set variables + self.id = self._get_param('id') self.cluster_id = self._get_param('cluster_id') self.name = self._get_param('name') self.load_demo_data = self._get_param('load_demo_data') @@ -148,76 +154,79 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - cluster = self.cdpy.dw.describe_cluster(cluster_id=self.cluster_id) - if cluster is None: - self.module.fail_json(msg="Couldn't retrieve cluster info for %s " % self.cluster_id) - else: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name) - # If Database Catalog exists - if self.target is not None: - if self.state == 'absent': - if self.module.check_mode: - self.clusters.append(self.target) + if self.id is None: + dbcs = self.cdpy.dw.list_dbcs(cluster_id=self.cluster_id) + for dbc in dbcs: + if self.name is not None and dbc['name'] == self.name: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc['id']) + elif self.id is not None and dbc['id'] == self.id: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) + # If Database Catalog exists + if self.target is not None: + if self.state == 'absent': + if self.module.check_mode: + self.dbcs.append(self.target) + else: + if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: + self.module.warn( + "DW Database Catalog not in valid state for Delete operation: %s" % self.target['status']) else: - if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn( - "DW Database Catalog not in valid state for Delete operation: %s" % self.target['status']) - else: - _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.name) - if self.wait: - self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_dbc, - params=dict(cluster_id=self.cluster_id, dbc_id=self.name), - field=None, delay=self.delay, timeout=self.timeout - ) - else: - self.cdpy.sdk.sleep(3) # Wait for consistency sync - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name) - self.clusters.append(self.target) - # Drop Done - elif self.state == 'present': - # Begin Config check - self.module.warn("DW Database Catalog already present and config validation is not implemented") + _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) if self.wait: - self.target = self.cdpy.sdk.wait_for_state( + self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, - params=dict(cluster_id=self.cluster_id,dbc_id=self.name), - state='Running', delay=self.delay, timeout=self.timeout + params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), + field=None, delay=self.delay, timeout=self.timeout ) - self.clusters.append(self.target) - # End Config check - else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) - # End handling Database Catalog exists - else: - # Begin handling Database Catalog not found - if self.state == 'absent': - self.module.warn("DW Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) - elif self.state == 'present': - if self.module.check_mode: - pass else: - self.name = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, - load_demo_data=self.load_demo_data) - if self.wait: - self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_dbc, - params=dict(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']), - state='Running', delay=self.delay, timeout=self.timeout - ) - else: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.name['dbcId']) + self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) self.dbcs.append(self.target) + # Drop Done + elif self.state == 'present': + # Begin Config check + self.module.warn("DW Database Catalog already present and config validation is not implemented") + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), + state='Running', delay=self.delay, timeout=self.timeout + ) + self.dbcs.append(self.target) + # End Config check + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) + # End handling Database Catalog exists + else: + # Begin handling Database Catalog not found + if self.state == 'absent': + self.module.warn("DW Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) + elif self.state == 'present': + if self.module.check_mode: + pass else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) + dbc_id = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, + load_demo_data=self.load_demo_data) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=dbc_id), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc_id) + self.dbcs.append(self.target) + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - cluster_id=dict(required=True, type='str', aliases=['cluster_id']), - name = dict(required=True, type='str', aliases=['name']), - load_demo_data=dict(required=False, type='bool', aliases=['load_demo_data']), + id=dict(required=False, type='str', default=None), + cluster_id=dict(required=True, type='str'), + name = dict(required=False, type='str', default=None), + load_demo_data=dict(required=False, type='bool', default=False), state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), wait = dict(required=False, type='bool', default=True), delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py index c39d3237..01e3835c 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_vw.py @@ -35,6 +35,11 @@ requirements: - cdpy options: + id: + description: + - If an ID is provided, that Virtual Warehouse will be deleted if C(state=absent) + type: str + required: When state is absent cluster_id: description: ID of cluster where Virtual Warehouse should be created. type: str @@ -217,7 +222,7 @@ # Delete Virtual Warehouse - cloudera.cloud.dw_vw: cluster_id: "example-cluster-id" - name: "example-virtual-warehouse" + id: "example-virtual-warehouse-id" state: absent ''' @@ -294,6 +299,7 @@ def __init__(self, module): super(DwVw, self).__init__(module) # Set variables + self.id = self._get_param('id') self.cluster_id = self._get_param('cluster_id') self.dbc_id = self._get_param('dbc_id') self.vw_type = self._get_param('vw_type') @@ -322,95 +328,97 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - cluster = self.cdpy.dw.describe_cluster(cluster_id=self.cluster_id) - if cluster is None: - self.module.fail_json(msg="Couldn't retrieve cluster info for %s " % self.cluster_id) - else: - dbc = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.dbc_id) - if dbc is None: - self.module.fail_json(msg="Couldn't retrieve dbc info for %s " % self.dbc_id) - else: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.name) - # If Virtual Warehouse exists - if self.target is not None: - if self.state == 'absent': - if self.module.check_mode: - self.clusters.append(self.target) - else: - if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn( - "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target[ - 'status']) - else: - _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.name) - if self.wait: - self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_vw, - params=dict(cluster_id=self.cluster_id, vw_id=self.name), - field=None, delay=self.delay, timeout=self.timeout - ) - else: - self.cdpy.sdk.sleep(3) # Wait for consistency sync - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.name) - self.clusters.append(self.target) - # Drop Done - elif self.state == 'present': - # Begin Config check - self.module.warn("DW Virtual Warehouse already present and config validation is not implemented") - if self.wait: - self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.delete_vw, - params=dict(cluster_id=self.cluster_id, vw_id=self.name), - state='Running', delay=self.delay, timeout=self.timeout - ) - self.clusters.append(self.target) - # End Config check - else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) - # End handling Virtual Warehouse exists + if self.id is None: + vws = self.cdpy.dw.list_vws(cluster_id=self.cluster_id) + for vw in vws: + if self.name is not None and vw['name'] == self.name: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw['id']) + elif self.id is not None and vw['id'] == self.id: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) + # If Virtual Warehouse exists + if self.target is not None: + if self.state == 'absent': + if self.module.check_mode: + self.vws.append(self.target) else: - # Begin handling Virtual Warehouse not found - if self.state == 'absent': + if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: self.module.warn( - "DW Virtual Warehouse %s already absent in Cluster %s" % (self.name, self.cluster_id)) - elif self.state == 'present': - if self.module.check_mode: - pass - else: - self.name = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, - dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, - template=self.template, autoscaling_min_cluster=self.autoscaling_min_cluster, - autoscaling_max_cluster=self.autoscaling_max_cluster, - common_configs=self.common_configs, application_configs=self.application_configs, - ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, tags=self.tags) - if self.wait: - self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_vw, - params=dict(cluster_id=self.cluster_id, vw_id=self.name), - state='Running', delay=self.delay, timeout=self.timeout - ) - else: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.vw_id) - self.vws.append(self.target) + "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target[ + 'status']) else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) + _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), + field=None, delay=self.delay, timeout=self.timeout + ) + else: + self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) + self.vws.append(self.target) + # Drop Done + elif self.state == 'present': + # Begin Config check + self.module.warn("DW Virtual Warehouse already present and config validation is not implemented") + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.delete_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), + state='Running', delay=self.delay, timeout=self.timeout + ) + self.vws.append(self.target) + # End Config check + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) + # End handling Virtual Warehouse exists + else: + # Begin handling Virtual Warehouse not found + if self.state == 'absent': + self.module.warn( + "DW Virtual Warehouse %s already absent in Cluster %s" % (self.name, self.cluster_id)) + elif self.state == 'present': + if self.module.check_mode: + pass + else: + vw_id = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, + dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, + template=self.template, + autoscaling_min_cluster=self.autoscaling_min_cluster, + autoscaling_max_cluster=self.autoscaling_max_cluster, + common_configs=self.common_configs, + application_configs=self.application_configs, + ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, + tags=self.tags) + if self.wait: + self.target = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=vw_id), + state='Running', delay=self.delay, timeout=self.timeout + ) + else: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw_id) + self.vws.append(self.target) + else: + self.module.fail_json(msg="State %s is not valid for this module" % self.state) def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - cluster_id=dict(required=True, type='str', aliases=['cluster_id']), - dbc_id=dict(required=False, type='str', aliases=['dbc_id']), - vw_type = dict(required=False, type='str', aliases=['vw_type']), - name = dict(required=True, type='str', aliases=['name']), - template=dict(required=False, type='str', aliases=['template']), - autoscaling_min_cluster=dict(required=False, type='int', aliases=['autoscaling_min_cluster']), - autoscaling_max_cluster=dict(required=False, type='int', aliases=['autoscaling_max_cluster']), - common_configs=dict(required=False, type='dict', aliases=['common_configs']), - application_configs=dict(required=False, type='dict', aliases=['application_configs']), - ldap_groups=dict(required=False, type='list', aliases=['ldap_groups']), - enable_sso=dict(required=False, type='bool', aliases=['enable_sso']), - tags=dict(required=False, type='dict', aliases=['tags']), + id=dict(required=False, type='str', default=None), + cluster_id=dict(required=True, type='str'), + dbc_id=dict(required=False, type='str', default=None), + vw_type = dict(required=False, type='str', default=None), + name = dict(required=False, type='str', default=None), + template=dict(required=False, type='str', default=None), + autoscaling_min_cluster=dict(required=False, type='int', default=None), + autoscaling_max_cluster=dict(required=False, type='int', default=None), + common_configs=dict(required=False, type='dict', default=None), + application_configs=dict(required=False, type='dict', default=None), + ldap_groups=dict(required=False, type='list', default=None), + enable_sso=dict(required=False, type='bool', default=False), + tags=dict(required=False, type='dict', default=None), state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), wait = dict(required=False, type='bool', default=True), delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), From 9e82a7031066bfd181e44e487d897863333d84e1 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 05/23] Use started state in wait for state function Signed-off-by: Saravanan Raju --- plugins/modules/dw_dbc.py | 4 ++-- plugins/modules/dw_vw.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_dbc.py index 41ef2555..8bb9d046 100644 --- a/plugins/modules/dw_dbc.py +++ b/plugins/modules/dw_dbc.py @@ -190,7 +190,7 @@ def process(self): self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), - state='Running', delay=self.delay, timeout=self.timeout + state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) self.dbcs.append(self.target) # End Config check @@ -211,7 +211,7 @@ def process(self): self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, params=dict(cluster_id=self.cluster_id, dbc_id=dbc_id), - state='Running', delay=self.delay, timeout=self.timeout + state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) else: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc_id) diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py index 01e3835c..bf19b8b9 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_vw.py @@ -365,7 +365,7 @@ def process(self): self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.delete_vw, params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), - state='Running', delay=self.delay, timeout=self.timeout + state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) self.vws.append(self.target) # End Config check @@ -394,7 +394,7 @@ def process(self): self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, params=dict(cluster_id=self.cluster_id, vw_id=vw_id), - state='Running', delay=self.delay, timeout=self.timeout + state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) else: self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw_id) From 0edfa73c755ae5b2c4e887686073eeb07fa102f0 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 06/23] Fix typo Signed-off-by: Saravanan Raju --- plugins/modules/dw_vw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_vw.py index bf19b8b9..4491c9b3 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_vw.py @@ -363,7 +363,7 @@ def process(self): self.module.warn("DW Virtual Warehouse already present and config validation is not implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.delete_vw, + describe_func=self.cdpy.dw.describe_vw, params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) From afbcde7164bb352f450a374373a8f6c335c9e5be Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 07/23] Rename vh and dbc module Signed-off-by: Saravanan Raju --- .../{dw_dbc.py => dw_database_catalog.py} | 22 +- .../{dw_vw.py => dw_virtual_warehouse.py} | 225 +++++++++--------- 2 files changed, 120 insertions(+), 127 deletions(-) rename plugins/modules/{dw_dbc.py => dw_database_catalog.py} (93%) rename plugins/modules/{dw_vw.py => dw_virtual_warehouse.py} (73%) diff --git a/plugins/modules/dw_dbc.py b/plugins/modules/dw_database_catalog.py similarity index 93% rename from plugins/modules/dw_dbc.py rename to plugins/modules/dw_database_catalog.py index 8bb9d046..bfecbe4e 100644 --- a/plugins/modules/dw_dbc.py +++ b/plugins/modules/dw_database_catalog.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module: dw_dbc +module: dw_database_catalog short_description: Create CDP Data Warehouse Database Catalog description: - Create CDP Database Catalog @@ -86,12 +86,12 @@ # Note: These examples do not set authentication details. # Create Database Catalog -- cloudera.cloud.dw_dbc: +- cloudera.cloud.dw_database_catalog: name: "example-database-catalog-name" cluster_id: "example-cluster-id" # Delete Database Catalog -- cloudera.cloud.dw_dbc: +- cloudera.cloud.dw_database_catalog: id: "example-database-id" cluster_id: "example-cluster-id" state: "absent" @@ -99,7 +99,7 @@ RETURN = r''' --- -dbcs: +database_catalogs: description: The information about the named Database Catalog. type: list returned: always @@ -144,7 +144,7 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.dbcs = [] + self.database_catalogs = [] # Initialize internal values self.target = None @@ -165,7 +165,7 @@ def process(self): if self.target is not None: if self.state == 'absent': if self.module.check_mode: - self.dbcs.append(self.target) + self.database_catalogs.append(self.target) else: if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: self.module.warn( @@ -179,9 +179,9 @@ def process(self): field=None, delay=self.delay, timeout=self.timeout ) else: - self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) - self.dbcs.append(self.target) + self.database_catalogs.append(self.target) # Drop Done elif self.state == 'present': # Begin Config check @@ -192,7 +192,7 @@ def process(self): params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) - self.dbcs.append(self.target) + self.database_catalogs.append(self.target) # End Config check else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) @@ -215,7 +215,7 @@ def process(self): ) else: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc_id) - self.dbcs.append(self.target) + self.database_catalogs.append(self.target) else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) @@ -236,7 +236,7 @@ def main(): ) result = DwDbc(module) - output = dict(changed=False, dbcs=result.dbcs) + output = dict(changed=False, database_catalogs=result.database_catalogs) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/dw_vw.py b/plugins/modules/dw_virtual_warehouse.py similarity index 73% rename from plugins/modules/dw_vw.py rename to plugins/modules/dw_virtual_warehouse.py index 4491c9b3..9ba59041 100644 --- a/plugins/modules/dw_vw.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module: dw_vw +module: dw_virtual_warehouse short_description: Create CDP Data Warehouse Virtual Warehouse description: - Create CDP Virtual Warehouse @@ -48,7 +48,7 @@ description: ID of Database Catalog that the Virtual Warehouse should be attached to. type: str required: True - vw_type: + type: description: Type of Virtual Warehouse to be created. type: str required: True @@ -60,105 +60,100 @@ description: Name of configuration template to use. type: str required: False - autoscaling_min_cluster: + autoscaling_min_nodes: description: Minimum number of available nodes for Virtual Warehouse autoscaling. type: int required: False - autoscaling_max_cluster: + autoscaling_max_nodes: description: Maximum number of available nodes for Virtual Warehouse autoscaling. type: int required: False - config: - description: Configuration settings for the Virtual Warehouse. + common_configs: + description: Configurations that are applied to every application in the service. type: dict required: False contains: - commonConfigs: - description: Configurations that are applied to every application in the service. - type: dict - required: False + configBlocks: List of ConfigBlocks for the application. + type: list + required: False contains: - configBlocks: List of ConfigBlocks for the application. - type: list - required: False - contains: - id: - description: ID of the ConfigBlock. Unique within an ApplicationConfig. - type: str - required: False - format: - description: Format of ConfigBlock. - type: str - required: False - content: - description: Contents of a ConfigBlock. - type: obj - required: False - contains: - keyValues: - description: Key-value type configurations. - type: obj - required: False - contains: - additionalProperties: - description: Key-value type configurations. - type: str - required: False - text: - description: Text type configuration. - type: str - required: False - json: - description: JSON type configuration. - type: str - required: False - applicationConfigs: - description: Application specific configurations. - type: dict - required: False - contains: - configBlocks: List of ConfigBlocks for the application. - type: list - required: False - contains: - id: - description: ID of the ConfigBlock. Unique within an ApplicationConfig. - type: str - required: False - format: - description: Format of ConfigBlock. - type: str - required: False - content: - description: Contents of a ConfigBlock. - type: obj - required: False - contains: - keyValues: - description: Key-value type configurations. - type: obj - required: False - contains: - additionalProperties: - description: Key-value type configurations. - type: str - required: False - text: - description: Text type configuration. - type: str - required: False - json: - description: JSON type configuration. - type: str - required: False - ldapGroups: - description: LDAP Groupnames to be enabled for auth. + id: + description: ID of the ConfigBlock. Unique within an ApplicationConfig. + type: str + required: False + format: + description: Format of ConfigBlock. + type: str + required: False + content: + description: Contents of a ConfigBlock. + type: obj + required: False + contains: + keyValues: + description: Key-value type configurations. + type: obj + required: False + contains: + additionalProperties: + description: Key-value type configurations. + type: str + required: False + text: + description: Text type configuration. + type: str + required: False + json: + description: JSON type configuration. + type: str + required: False + application_configs: + description: Configurations that are applied to every application in the service. + type: dict + required: False + contains: + configBlocks: List of ConfigBlocks for the application. type: list - required: False - enableSSO: - description: Should SSO be enabled for this VW. - type: bool - required: False + required: False + contains: + id: + description: ID of the ConfigBlock. Unique within an ApplicationConfig. + type: str + required: False + format: + description: Format of ConfigBlock. + type: str + required: False + content: + description: Contents of a ConfigBlock. + type: obj + required: False + contains: + keyValues: + description: Key-value type configurations. + type: obj + required: False + contains: + additionalProperties: + description: Key-value type configurations. + type: str + required: False + text: + description: Text type configuration. + type: str + required: False + json: + description: JSON type configuration. + type: str + required: False + ldap_groups: + description: LDAP Groupnames to be enabled for auth. + type: list + required: False + enable_sso: + description: Should SSO be enabled for this VW. + type: bool + required: False tags: description: Tags associated with the resources. type: dict @@ -205,22 +200,20 @@ # Note: These examples do not set authentication details. # Create Virtual Warehouse -- cloudera.cloud.dw_vw: +- cloudera.cloud.dw_virtual_warehouse: cluster_id: "example-cluster-id" name: "example-virtual-warehouse" - vw_type: "hive" + type: "hive" template: "xsmall" - autoscaling: - min_cluster: 3 - max_cluster: 19 + autoscaling_min_nodes: 3 + autoscaling_max_nodes: 19 tags: tag-key: "tag-value" - configs: - enable_sso: true - ldap_groups: ['group1','group2','group3'] + enable_sso: true + ldap_groups: ['group1','group2','group3'] # Delete Virtual Warehouse -- cloudera.cloud.dw_vw: +- cloudera.cloud.dw_virtual_warehouse: cluster_id: "example-cluster-id" id: "example-virtual-warehouse-id" state: absent @@ -228,7 +221,7 @@ RETURN = r''' --- -vws: +virtual_warehouses: description: The information about the named CDW Virtual Warehouses. type: list returned: always @@ -302,11 +295,11 @@ def __init__(self, module): self.id = self._get_param('id') self.cluster_id = self._get_param('cluster_id') self.dbc_id = self._get_param('dbc_id') - self.vw_type = self._get_param('vw_type') + self.type = self._get_param('type') self.name = self._get_param('name') self.template = self._get_param('template') - self.autoscaling_min_cluster = self._get_param('autoscaling_min_cluster') - self.autoscaling_max_cluster = self._get_param('autoscaling_max_cluster') + self.autoscaling_min_nodes = self._get_param('autoscaling_min_nodes') + self.autoscaling_max_nodes = self._get_param('autoscaling_max_nodes') self.common_configs = self._get_param('common_configs') self.application_configs = self._get_param('application_configs') self.ldap_groups = self._get_param('ldap_groups') @@ -318,7 +311,7 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.vws = [] + self.virtual_warehouses = [] # Initialize internal values self.target = None @@ -339,7 +332,7 @@ def process(self): if self.target is not None: if self.state == 'absent': if self.module.check_mode: - self.vws.append(self.target) + self.virtual_warehouses.append(self.target) else: if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: self.module.warn( @@ -354,9 +347,9 @@ def process(self): field=None, delay=self.delay, timeout=self.timeout ) else: - self.cdpy.sdk.sleep(3) # Wait for consistency sync + self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) - self.vws.append(self.target) + self.virtual_warehouses.append(self.target) # Drop Done elif self.state == 'present': # Begin Config check @@ -367,7 +360,7 @@ def process(self): params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) - self.vws.append(self.target) + self.virtual_warehouses.append(self.target) # End Config check else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) @@ -382,10 +375,10 @@ def process(self): pass else: vw_id = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, - dbc_id=self.dbc_id, vw_type=self.vw_type, name=self.name, + dbc_id=self.dbc_id, vw_type=self.type, name=self.name, template=self.template, - autoscaling_min_cluster=self.autoscaling_min_cluster, - autoscaling_max_cluster=self.autoscaling_max_cluster, + autoscaling_min_cluster=self.autoscaling_min_nodes, + autoscaling_max_cluster=self.autoscaling_max_nodes, common_configs=self.common_configs, application_configs=self.application_configs, ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, @@ -398,7 +391,7 @@ def process(self): ) else: self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw_id) - self.vws.append(self.target) + self.virtual_warehouses.append(self.target) else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) @@ -409,11 +402,11 @@ def main(): id=dict(required=False, type='str', default=None), cluster_id=dict(required=True, type='str'), dbc_id=dict(required=False, type='str', default=None), - vw_type = dict(required=False, type='str', default=None), + type = dict(required=False, type='str', default=None), name = dict(required=False, type='str', default=None), template=dict(required=False, type='str', default=None), - autoscaling_min_cluster=dict(required=False, type='int', default=None), - autoscaling_max_cluster=dict(required=False, type='int', default=None), + autoscaling_min_nodes=dict(required=False, type='int', default=None), + autoscaling_max_nodes=dict(required=False, type='int', default=None), common_configs=dict(required=False, type='dict', default=None), application_configs=dict(required=False, type='dict', default=None), ldap_groups=dict(required=False, type='list', default=None), @@ -428,7 +421,7 @@ def main(): ) result = DwVw(module) - output = dict(changed=False, vws=result.vws) + output = dict(changed=False, virtual_warehouses=result.virtual_warehouses) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) From babd10e05afb5c326bda4bd5aa198b14077c1286 Mon Sep 17 00:00:00 2001 From: Saravanan Raju Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 08/23] Expand common_config and application_config Signed-off-by: Saravanan Raju --- plugins/modules/dw_database_catalog.py | 21 +-- plugins/modules/dw_virtual_warehouse.py | 188 +++++++++++++++--------- 2 files changed, 134 insertions(+), 75 deletions(-) diff --git a/plugins/modules/dw_database_catalog.py b/plugins/modules/dw_database_catalog.py index bfecbe4e..8bee6c68 100644 --- a/plugins/modules/dw_database_catalog.py +++ b/plugins/modules/dw_database_catalog.py @@ -104,7 +104,7 @@ type: list returned: always elements: complex - contains: + suboptions: id: description: The id of the Database Catalog. returned: always @@ -148,6 +148,7 @@ def __init__(self, module): # Initialize internal values self.target = None + self.changed = False # Execute logic process self.process() @@ -172,6 +173,7 @@ def process(self): "DW Database Catalog not in valid state for Delete operation: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) + self.changed = True if self.wait: self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, @@ -207,6 +209,7 @@ def process(self): else: dbc_id = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, load_demo_data=self.load_demo_data) + self.changed = True if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, @@ -223,20 +226,20 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(required=False, type='str', default=None), + id=dict(type='str'), cluster_id=dict(required=True, type='str'), - name = dict(required=False, type='str', default=None), - load_demo_data=dict(required=False, type='bool', default=False), - state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), - wait = dict(required=False, type='bool', default=True), - delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), - timeout = dict(required=False, type='int', aliases=['polling_timeout'], default=3600) + name = dict(type='str'), + load_demo_data=dict(type='bool'), + state=dict(type='str', choices=['present', 'absent'], default='present'), + wait = dict(type='bool', default=True), + delay = dict(type='int', aliases=['polling_delay'], default=15), + timeout = dict(type='int', aliases=['polling_timeout'], default=3600) ), supports_check_mode=True ) result = DwDbc(module) - output = dict(changed=False, database_catalogs=result.database_catalogs) + output = dict(changed=result.changed, database_catalogs=result.database_catalogs) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index 9ba59041..88dbe812 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -72,11 +72,12 @@ description: Configurations that are applied to every application in the service. type: dict required: False - contains: + suboptions: configBlocks: List of ConfigBlocks for the application. type: list required: False - contains: + elements: dict + suboptions: id: description: ID of the ConfigBlock. Unique within an ApplicationConfig. type: str @@ -84,21 +85,16 @@ format: description: Format of ConfigBlock. type: str - required: False + required: False content: description: Contents of a ConfigBlock. - type: obj + type: dict required: False - contains: + suboptions: keyValues: description: Key-value type configurations. - type: obj + type: dict required: False - contains: - additionalProperties: - description: Key-value type configurations. - type: str - required: False text: description: Text type configuration. type: str @@ -111,45 +107,45 @@ description: Configurations that are applied to every application in the service. type: dict required: False - contains: - configBlocks: List of ConfigBlocks for the application. - type: list - required: False - contains: - id: - description: ID of the ConfigBlock. Unique within an ApplicationConfig. - type: str - required: False - format: - description: Format of ConfigBlock. - type: str - required: False - content: - description: Contents of a ConfigBlock. - type: obj - required: False - contains: - keyValues: - description: Key-value type configurations. - type: obj - required: False - contains: - additionalProperties: - description: Key-value type configurations. - type: str - required: False - text: - description: Text type configuration. - type: str - required: False - json: - description: JSON type configuration. - type: str - required: False + suboptions: + required: False + type: dict + suboptions: + configBlocks: List of ConfigBlocks for the application. + type: list + required: False + elements: dict + suboptions: + id: + description: ID of the ConfigBlock. Unique within an ApplicationConfig. + type: str + required: False + format: + description: Format of ConfigBlock. + type: str + required: False + content: + description: Contents of a ConfigBlock. + type: dict + required: False + suboptions: + keyValues: + description: Key-value type configurations. + type: dict + required: False + text: + description: Text type configuration. + type: str + required: False + json: + description: JSON type configuration. + type: str + required: False ldap_groups: description: LDAP Groupnames to be enabled for auth. type: list required: False + elements: str enable_sso: description: Should SSO be enabled for this VW. type: bool @@ -211,6 +207,41 @@ tag-key: "tag-value" enable_sso: true ldap_groups: ['group1','group2','group3'] + +- cloudera.cloud.dw_virtual_warehouse: + cluster_id: "example-cluster-id" + name: "example-virtual-warehouse" + type: "hive" + template: "xsmall" + enable_sso: true + ldap_groups: ['group1','group2','group3'] + common_configs: "{ + 'configBlocks': [ + { + 'id': 'das-ranger-policymgr', + 'format': 'HADOOP_XML', + 'content': { + 'keyValues' : { + 'xasecure.policymgr.clientssl.truststore': '/path_to_ca_cert/cacerts' + } + } + } + ] + }" + application_configs: "{ + "das-webapp": { + "configBlocks": [ + { + "id": "hive-kerberos-config", + "format": "TEXT", + "content": { + "text": "\n[libdefaults]\n\trenew_lifetime = 7d" + } + } + ] + } + }" + # Delete Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: @@ -226,10 +257,10 @@ type: list returned: always elements: complex - contains: + suboptions: vws: type: dict - contains: + suboptions: id: description: Id of the Virtual Warehouse created. returned: always @@ -258,7 +289,7 @@ description: The CRN of the cluster creator. returned: always type: dict - contains: + suboptions: crn: type: str description: Actor CRN @@ -315,6 +346,7 @@ def __init__(self, module): # Initialize internal values self.target = None + self.changed = False # Execute logic process self.process() @@ -340,6 +372,7 @@ def process(self): 'status']) else: _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) + self.changed = True if self.wait: self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, @@ -383,6 +416,7 @@ def process(self): application_configs=self.application_configs, ldap_groups=self.ldap_groups, enable_sso=self.enable_sso, tags=self.tags) + self.changed = True if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, @@ -399,29 +433,51 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(required=False, type='str', default=None), + id=dict(type='str'), cluster_id=dict(required=True, type='str'), - dbc_id=dict(required=False, type='str', default=None), - type = dict(required=False, type='str', default=None), - name = dict(required=False, type='str', default=None), - template=dict(required=False, type='str', default=None), - autoscaling_min_nodes=dict(required=False, type='int', default=None), - autoscaling_max_nodes=dict(required=False, type='int', default=None), - common_configs=dict(required=False, type='dict', default=None), - application_configs=dict(required=False, type='dict', default=None), - ldap_groups=dict(required=False, type='list', default=None), - enable_sso=dict(required=False, type='bool', default=False), - tags=dict(required=False, type='dict', default=None), - state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), - wait = dict(required=False, type='bool', default=True), - delay = dict(required=False, type='int', aliases=['polling_delay'], default=15), - timeout = dict(required=False, type='int', aliases=['polling_timeout'], default=3600) + dbc_id=dict(type='str'), + type = dict(type='str'), + name = dict(type='str'), + template=dict(type='str'), + autoscaling_min_nodes=dict(type='int'), + autoscaling_max_nodes=dict(type='int'), + common_configs=dict(type='dict', + options=dict( + configBlocks = dict( + type='list', + elements='dict', + options=dict( + dict( + id=dict(type='str'), + format=dict(type='str', choices=['HADOOP_XML', 'PROPERTIES', 'TEXT', 'JSON', 'BINARY', 'ENV', 'FLAGFILE']), + content=dict(type='dict', + options=dict( + dict( + keyValues=dict(type='dict'), + text=dict(type='str'), + json=dict(type='json') + ) + ) + ) + ) + ) + ) + ) + ), + application_configs=dict(type='dict'), + ldap_groups=dict(type='list'), + enable_sso=dict(type='bool'), + tags=dict(type='dict'), + state=dict(type='str', choices=['present', 'absent'], default='present'), + wait = dict(type='bool', default=True), + delay = dict(type='int', aliases=['polling_delay'], default=15), + timeout = dict(type='int', aliases=['polling_timeout'], default=3600) ), supports_check_mode=True ) result = DwVw(module) - output = dict(changed=False, virtual_warehouses=result.virtual_warehouses) + output = dict(changed=result.changed, virtual_warehouses=result.virtual_warehouses) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) From 29b001423fc77feda2e9fe8d8dfe546e7013b00d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:40 -0400 Subject: [PATCH 09/23] Update module documentation, return values, and logic flow Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster.py | 238 +++++++++++++------------ plugins/modules/dw_database_catalog.py | 96 +++++----- 2 files changed, 175 insertions(+), 159 deletions(-) diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index a38fa10d..9826d774 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -30,53 +30,61 @@ - Create or Delete CDP Data Warehouse Clusters author: - "Dan Chaffelson (@chaffelson)" + - "Saravanan Raju (@raju-saravanan)" + - "Webster Mudge (@wmudge)" requirements: - cdpy options: - id: + name: description: - - If an ID is provided, that Data Warehouse Cluster will be deleted if C(state=absent) + - The identifier of the Data Warehouse Cluster. + - Required if C(state=absent) and C(env) is not specified. type: str - required: When state is absent aliases: - - name + - id env: - description: The name of the target environment + description: + - The name of the target environment. + - Required if C(state=present) or C(state=absent) and C(id) is not specified. type: str - required: when state is present aliases: - environment - env_crn overlay: - description: Set it to true to save IP addresses in the VPC by using a private IP address range for Pods in the cluster. + description: + - Flag to use private IP addresses for Pods within the cluster. + - Otherwise, use IP addresses within the VPC. type: bool - required: False default: False private_load_balancer: - description: Set up load balancer in private subnets. + description: Flag to set up a load balancer for private subnets. type: bool - required: False default: False aws_public_subnets: - description: List of zero or more Public AWS Subnet IDs to deploy to + description: + - List of zero or more Public AWS Subnet IDs used for deployment. + - Required if C(state=present) and the C(env) is deployed to AWS. type: list - required: when state is present for AWS deployment + elements: str aws_private_subnets: - description: List of zero or more Private AWS Subnet IDs to deploy to + description: + - List of zero or more Private AWS Subnet IDs used for deployment. + - Required if C(state=present) and the C(env) is deployed to AWS. type: list - required: when state is present for AWS deployment + elements: str az_subnet: - description: Azure Subnet Name, not URI + description: + - The Azure Subnet Name. + - Required if C(state=present) and the C(env) is deployed to Azure. type: str - required: when state is present for Azure deploymnet az_enable_az: - description: Whether to enable AZ mode or not + description: + - Flag to enable Availability Zone mode. + - Required if C(state=present) and the C(env) is deployed to Azure. type: bool - required: when state is present for Azure deployment state: - description: The declarative state of the Data Warehouse Cluster + description: The state of the Data Warehouse Cluster type: str - required: False default: present choices: - present @@ -86,14 +94,18 @@ - Flag to enable internal polling to wait for the Data Warehouse Cluster to achieve the declared state. - If set to FALSE, the module will return immediately. type: bool - required: False default: True + force: + description: + - Flag to enable force deletion of the Data Warehouse Cluster. + - This will not destroy the underlying cloud provider assets. + type: bool + default: False delay: description: - The internal polling interval (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared state. type: int - required: False default: 15 aliases: - polling_delay @@ -102,7 +114,6 @@ - The internal polling timeout (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared state. type: int - required: False default: 3600 aliases: - polling_timeout @@ -114,11 +125,6 @@ EXAMPLES = r''' # Note: These examples do not set authentication details. -# Delete a Data Warehouse Cluster -- cloudera.cloud.dw_cluster: - state: absent - id: my-id - # Request Azure Cluster creation - cloudera.cloud.dw_cluster: env_crn: crn:cdp:environments... @@ -130,60 +136,65 @@ env_crn: crn:cdp:environments... aws_public_subnets: [subnet-id-1, subnet-id-2] aws_private_subnets: [subnet-id-3, subnet-id-4] + +# Delete a Data Warehouse Cluster +- cloudera.cloud.dw_cluster: + state: absent + id: my-id + +# Delete the Data Warehouse Cluster within the Environment +- cloudera.cloud.dw_cluster: + state: absent + env: crn:cdp:environments... ''' RETURN = r''' --- -clusters: - description: The information about the named Cluster or Clusters - type: list - returned: always - elements: complex +cluster: + description: Details for the Data Warehouse cluster + type: dict contains: - cluster: + name: + description: The name of the cluster. + returned: always + type: str + environmentCrn: + description: The crn of the cluster's environment. + returned: always + type: str + crn: + description: The cluster's CRN. + returned: always + type: str + creationDate: + description: The creation time of the cluster in UTC. + returned: always + type: str + status: + description: The status of the cluster. + returned: always + type: str + creator: + description: The details regarding the cluster creator. + returned: always type: dict contains: - name: - description: The name of the cluster. - returned: always - type: str - environmentCrn: - description: The crn of the cluster's environment. - returned: always - type: str crn: - description: The cluster's crn. - returned: always type: str - creationDate: - description: The creation time of the cluster in UTC. - returned: always + description: The creator's CRN + email: type: str - status: - description: The status of the Cluster - returned: always + description: Email address + workloadUsername: type: str - creator: - description: The CRN of the cluster creator. - returned: always - type: dict - contains: - crn: - type: str - description: Actor CRN - email: - type: str - description: Email address for users - workloadUsername: - type: str - description: Username for users - machineUsername: - type: str - description: Username for machine users - cloudPlatform: - description: The cloud platform of the environment that was used to create this cluster - returned: always + description: Username + machineUsername: type: str + description: Username for machine users + cloudPlatform: + description: The cloud platform of the environment housing the Data Warehouse cluster + returned: always + type: str sdk_out: description: Returns the captured CDP SDK log. returned: when supported @@ -216,7 +227,8 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.clusters = [] + self.cluster = {} + self.changed = False # Initialize internal values self.target = None @@ -227,6 +239,7 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): env_crn = self.cdpy.environments.resolve_environment_crn(self.env) + # Check if Cluster exists if self.name is not None: self.target = self.cdpy.dw.describe_cluster(cluster_id=self.name) @@ -238,22 +251,23 @@ def process(self): elif len(listing) == 0: self.target = None else: - self.module.fail_json( - msg="Got ambiguous listing result for DW Clusters in Environment %s" % self.env) + self.module.fail_json(msg="Received multiple (i.e. ambiguious) DW Clusters in Environment %s" % self.env) else: self.target = None + if self.target is not None: - # Cluster Exists + # Begin Cluster Exists if self.state == 'absent': # Begin Delete if self.module.check_mode: - self.clusters.append(self.target) + self.cluster = self.target else: + self.changed = True if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn( - "DW Cluster not in valid state for Delete operation: %s" % self.target['status']) + self.module.warn("DW Cluster is not in a valid state for Delete operations: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_cluster(cluster_id=self.name, force=self.force) + if self.wait: self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_cluster, @@ -261,33 +275,31 @@ def process(self): field=None, delay=self.delay, timeout=self.timeout ) else: - self.cdpy.sdk.sleep(3) # Wait for consistency sync - self.target = self.cdpy.dw.describe_cluster(cluster_id=self.name) - self.clusters.append(self.target) - # Drop Done + self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync + self.cluster = self.cdpy.dw.describe_cluster(cluster_id=self.name) + # End Delete elif self.state == 'present': - # Being Config check - self.module.warn("DW Cluster already present and config validation is not implemented") + # Begin Config Check + self.module.warn("DW Cluster is already present and reconciliation is not yet implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_cluster, params=dict(cluster_id=self.name), state='Running', delay=self.delay, timeout=self.timeout ) - self.clusters.append(self.target) - # End Config check + self.cluster = self.target + # End Config Check else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) - # End handling Cluster exists + # End Cluster Exists else: - # Begin handling Cluster not found + # Begin Cluster Not Found if self.state == 'absent': - self.module.warn("DW CLuster %s already absent in Environment %s" % (self.name, self.env)) + self.module.warn("DW Cluster %s already absent in Environment %s" % (self.name, self.env)) elif self.state == 'present': - if self.module.check_mode: - pass - else: - # Being handle Cluster Creation + if not self.module.check_mode: + # Begin Cluster Creation + self.changed = True if env_crn is None: self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) else: @@ -297,45 +309,49 @@ def process(self): az_subnet=self.az_subnet, az_enable_az=self.az_enable_az ) if self.wait: - self.target = self.cdpy.sdk.wait_for_state( + self.cluster = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_cluster, params=dict(cluster_id=self.name), state='Running', delay=self.delay, timeout=self.timeout ) else: - self.target = self.cdpy.dw.describe_cluster(cluster_id=self.name) - self.clusters.append(self.target) + self.cluster = self.cdpy.dw.describe_cluster(cluster_id=self.name) + # End Cluster Creation else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) + self.module.fail_json(msg="Invalid state: %s" % self.state) + # End Cluster Not Found def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - name=dict(required=False, type='str', aliases=['id']), - env=dict(required=False, type='str', aliases=['environment', 'env_crn']), - overlay=dict(required=False, type='bool', default=False), - private_load_balancer=dict(required=False, type='bool', default=False), - az_subnet=dict(required=False, type='str', default=None), - az_enable_az=dict(required=False, type='bool', default=None), - aws_public_subnets=dict(required=False, type='list', default=None), - aws_private_subnets=dict(required=False, type='list', default=None), - state=dict(required=False, type='str', choices=['present', 'absent'], default='present'), - force=dict(required=False, type='bool', default=False), - wait=dict(required=False, type='bool', default=True), - delay=dict(required=False, type='int', aliases=['polling_delay'], default=15), - timeout=dict(required=False, type='int', aliases=['polling_timeout'], default=3600) + name=dict(type='str', aliases=['id']), + env=dict(type='str', aliases=['environment', 'env_crn']), + overlay=dict(type='bool', default=False), + private_load_balancer=dict(type='bool', default=False), + az_subnet=dict(type='str', default=None), + az_enable_az=dict(type='bool', default=None), + aws_public_subnets=dict(type='list', default=None), + aws_private_subnets=dict(type='list', default=None), + state=dict(type='str', choices=['present', 'absent'], default='present'), + force=dict(type='bool', default=False), + wait=dict(type='bool', default=True), + delay=dict(type='int', aliases=['polling_delay'], default=15), + timeout=dict(type='int', aliases=['polling_timeout'], default=3600) ), required_together=[ ['az_subnet', 'az_enable_az'], ['aws_public_subnets', 'aws_private_subnets'] ], - required_one_of=[['name', 'env'], ], + required_if=[ + ['state', 'absent', ['name', 'env']], + ['state', 'present', ['env']] + ], supports_check_mode=True ) result = DwCluster(module) - output = dict(changed=False, clusters=result.clusters) + output = dict(changed=result.changed, cluster=result.cluster) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/dw_database_catalog.py b/plugins/modules/dw_database_catalog.py index 8bee6c68..af74564b 100644 --- a/plugins/modules/dw_database_catalog.py +++ b/plugins/modules/dw_database_catalog.py @@ -25,9 +25,9 @@ DOCUMENTATION = r''' --- module: dw_database_catalog -short_description: Create CDP Data Warehouse Database Catalog +short_description: Create, manage, and destroy CDP Data Warehouse Database Catalogs description: - - Create CDP Database Catalog + - Create, manage, and destroy CDP Data Warehouse Database Catalogs author: - "Webster Mudge (@wmudge)" - "Dan Chaffelson (@chaffelson)" @@ -37,34 +37,34 @@ options: id: description: - - If an ID is provided, that Database Catalog will be deleted if C(state=absent) + - The identifier of the Database Catalog. + - Required if C(state=absent). type: str - required: When state is absent cluster_id: - description: ID of cluster where Database Catalog should be created. + description: + - The identifier of the parent DW Cluster of the Database Catalog. type: str required: True name: - description: Name of the Database Catalog. + description: + - The name of the Database Catalog. + - Required if C(state=present). type: str - required: True load_demo_data: - description: Set this to true if you demo data should be loaded into the Database Catalog. + description: + - Flag to load demonstration data into the Database Catalog during creation. type: str - required: False wait: description: - Flag to enable internal polling to wait for the Data Catalog to achieve the declared state. - If set to FALSE, the module will return immediately. type: bool - required: False default: True delay: description: - The internal polling interval (in seconds) while the module waits for the Data Catalog to achieve the declared state. type: int - required: False default: 15 aliases: - polling_delay @@ -73,7 +73,6 @@ - The internal polling timeout (in seconds) while the module waits for the Data Catalog to achieve the declared state. type: int - required: False default: 3600 aliases: - polling_timeout @@ -87,26 +86,25 @@ # Create Database Catalog - cloudera.cloud.dw_database_catalog: - name: "example-database-catalog-name" - cluster_id: "example-cluster-id" + name: example-database-catalog-name + cluster_id: example-cluster-id # Delete Database Catalog - cloudera.cloud.dw_database_catalog: - id: "example-database-id" - cluster_id: "example-cluster-id" - state: "absent" + id: example-database-id + cluster_id: example-cluster-id + state: absent ''' RETURN = r''' --- -database_catalogs: - description: The information about the named Database Catalog. - type: list +database_catalog: + description: Details about the Database Catalog. returned: always - elements: complex - suboptions: + type: dict + contains: id: - description: The id of the Database Catalog. + description: The identifier of the Database Catalog. returned: always type: str name: @@ -144,11 +142,11 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.database_catalogs = [] + self.database_catalog = {} + self.changed = False # Initialize internal values self.target = None - self.changed = False # Execute logic process self.process() @@ -160,17 +158,19 @@ def process(self): for dbc in dbcs: if self.name is not None and dbc['name'] == self.name: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc['id']) - elif self.id is not None and dbc['id'] == self.id: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) - # If Database Catalog exists + else: + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) + if self.target is not None: + # Begin Database Catalog Exists if self.state == 'absent': if self.module.check_mode: - self.database_catalogs.append(self.target) + self.database_catalog = self.target else: + # Begin Drop if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: self.module.warn( - "DW Database Catalog not in valid state for Delete operation: %s" % self.target['status']) + "DW Database Catalog is not in a valid state for Delete operations: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) self.changed = True @@ -182,46 +182,42 @@ def process(self): ) else: self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) - self.database_catalogs.append(self.target) - # Drop Done + self.database_catalog = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) + # End Drop elif self.state == 'present': - # Begin Config check - self.module.warn("DW Database Catalog already present and config validation is not implemented") + # Begin Config Check + self.module.warn("DW Database Catalog already present and reconciliation is not yet implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) - self.database_catalogs.append(self.target) - # End Config check + self.database_catalog = self.target + # End Config Check else: - self.module.fail_json(msg="State %s is not valid for this module" % self.state) - # End handling Database Catalog exists + self.module.fail_json(msg="Invalid state %s" % self.state) + # End Database Catalog Exists else: - # Begin handling Database Catalog not found + # Begin Database Catalog Not Found if self.state == 'absent': self.module.warn("DW Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) elif self.state == 'present': - if self.module.check_mode: - pass - else: + if not self.module.check_mode: dbc_id = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, load_demo_data=self.load_demo_data) self.changed = True if self.wait: - self.target = self.cdpy.sdk.wait_for_state( + self.database_catalog = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, params=dict(cluster_id=self.cluster_id, dbc_id=dbc_id), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) else: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc_id) - self.database_catalogs.append(self.target) + self.database_catalog = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc_id) else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) - + # End Database Catalog Not Found def main(): module = AnsibleModule( @@ -235,11 +231,15 @@ def main(): delay = dict(type='int', aliases=['polling_delay'], default=15), timeout = dict(type='int', aliases=['polling_timeout'], default=3600) ), + required_if=[ + ['state', 'present', ['name']], + ['state', 'absent', ['name', 'id']], + ], supports_check_mode=True ) result = DwDbc(module) - output = dict(changed=result.changed, database_catalogs=result.database_catalogs) + output = dict(changed=result.changed, database_catalog=result.database_catalog) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) From 81e84245388c8407f1951b521c3fd8c628702971 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:41 -0400 Subject: [PATCH 10/23] WIP of documentation updates, return values and logic flow Signed-off-by: Webster Mudge --- plugins/modules/dw_virtual_warehouse.py | 289 +++++++++++------------- 1 file changed, 134 insertions(+), 155 deletions(-) diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index 88dbe812..8b108ad4 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -25,7 +25,7 @@ DOCUMENTATION = r''' --- module: dw_virtual_warehouse -short_description: Create CDP Data Warehouse Virtual Warehouse +short_description: Create, manage, and destroy CDP Data Warehouse Virtual Warehouses description: - Create CDP Virtual Warehouse author: @@ -37,153 +37,146 @@ options: id: description: - - If an ID is provided, that Virtual Warehouse will be deleted if C(state=absent) + - The identifier of the Virtual Warehouse. + - Required if C(state=absent). type: str - required: When state is absent cluster_id: - description: ID of cluster where Virtual Warehouse should be created. + description: + - The identifier of the parent DW Cluster of the Virtual Warehouse. type: str required: True dbc_id: - description: ID of Database Catalog that the Virtual Warehouse should be attached to. + description: + - The identifier of the parent Database Catalog attached to the Virtual Warehouse. type: str required: True type: - description: Type of Virtual Warehouse to be created. + description: + - The type of Virtual Warehouse to be created. + - Required if C(state=present) type: str - required: True + choices: + - hive + - impala name: - description: Name of the Virtual Warehouse. + description: + - The name of the Virtual Warehouse. + - Required if C(state=present) type: str - required: True template: - description: Name of configuration template to use. + description: The name of deployment template for the Virtual Warehouse type: str - required: False + choices: + - xsmall + - small + - medium + - large autoscaling_min_nodes: - description: Minimum number of available nodes for Virtual Warehouse autoscaling. + description: The minimum number of available nodes for Virtual Warehouse autoscaling. type: int - required: False autoscaling_max_nodes: - description: Maximum number of available nodes for Virtual Warehouse autoscaling. + description: The maximum number of available nodes for Virtual Warehouse autoscaling. type: int - required: False common_configs: - description: Configurations that are applied to every application in the service. + description: Configurations that are applied to every application in the Virtual Warehouse service. type: dict - required: False suboptions: - configBlocks: List of ConfigBlocks for the application. + configBlocks: List of I(ConfigBlocks) for the application. type: list - required: False elements: dict suboptions: id: - description: ID of the ConfigBlock. Unique within an ApplicationConfig. + description: + - ID of the ConfigBlock. + - Unique within an I(ApplicationConfig). type: str - required: False format: - description: Format of ConfigBlock. + description: Format of the ConfigBlock. type: str - required: False content: - description: Contents of a ConfigBlock. + description: Contents of the ConfigBlock. type: dict - required: False suboptions: keyValues: - description: Key-value type configurations. + description: Key-value type configuration. type: dict - required: False text: description: Text type configuration. type: str - required: False json: description: JSON type configuration. type: str - required: False application_configs: - description: Configurations that are applied to every application in the service. + description: Configurations that are applied to specific applications in the Virtual Warehouse service. type: dict - required: False suboptions: - required: False - type: dict - suboptions: - configBlocks: List of ConfigBlocks for the application. - type: list - required: False - elements: dict - suboptions: - id: - description: ID of the ConfigBlock. Unique within an ApplicationConfig. - type: str - required: False - format: - description: Format of ConfigBlock. - type: str - required: False - content: - description: Contents of a ConfigBlock. - type: dict - required: False - suboptions: - keyValues: - description: Key-value type configurations. - type: dict - required: False - text: - description: Text type configuration. - type: str - required: False - json: - description: JSON type configuration. - type: str - required: False + __application_name__: + description: The application name or identifier. + type: dict + suboptions: + configBlocks: List of ConfigBlocks for the application. + type: list + required: False + elements: dict + suboptions: + id: + description: + - ID of the ConfigBlock. + - Unique within an ApplicationConfig. + type: str + format: + description: Format of ConfigBlock. + type: str + content: + description: Contents of a ConfigBlock. + type: dict + suboptions: + keyValues: + description: Key-value type configuration. + type: dict + text: + description: Text type configuration. + type: str + json: + description: JSON type configuration. + type: str ldap_groups: - description: LDAP Groupnames to be enabled for auth. + description: LDAP Groupnames to enabled for authentication to the Virtual Warehouse. type: list - required: False elements: str enable_sso: - description: Should SSO be enabled for this VW. + description: Flag to enable SSO for the Virtual Warehouse. type: bool - required: False + default: False tags: - description: Tags associated with the resources. + description: Key-value tags associated with the Virtual Warehouse cloud provider resources. type: dict - required: False state: description: The declarative state of the Virtual Warehouse type: str - required: False default: present choices: - present - absent wait: description: - - Flag to enable internal polling to wait for the Data Warehouse Cluster to achieve the declared state. + - Flag to enable internal polling to wait for the Virtual Warehouse to achieve the declared state. - If set to FALSE, the module will return immediately. type: bool - required: False default: True delay: description: - - The internal polling interval (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared + - The internal polling interval (in seconds) while the module waits for the Virtual Warehouse to achieve the declared state. type: int - required: False default: 15 aliases: - polling_delay timeout: description: - - The internal polling timeout (in seconds) while the module waits for the Data Warehouse Cluster to achieve the declared + - The internal polling timeout (in seconds) while the module waits for the Virtual Warehouse to achieve the declared state. type: int - required: False default: 3600 aliases: - polling_timeout @@ -197,10 +190,10 @@ # Create Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: - cluster_id: "example-cluster-id" - name: "example-virtual-warehouse" - type: "hive" - template: "xsmall" + cluster_id: example-cluster-id + name: example-virtual-warehouse + type: hive + template: xsmall autoscaling_min_nodes: 3 autoscaling_max_nodes: 19 tags: @@ -215,34 +208,21 @@ template: "xsmall" enable_sso: true ldap_groups: ['group1','group2','group3'] - common_configs: "{ - 'configBlocks': [ - { - 'id': 'das-ranger-policymgr', - 'format': 'HADOOP_XML', - 'content': { - 'keyValues' : { - 'xasecure.policymgr.clientssl.truststore': '/path_to_ca_cert/cacerts' - } - } - } - ] - }" - application_configs: "{ - "das-webapp": { - "configBlocks": [ - { - "id": "hive-kerberos-config", - "format": "TEXT", - "content": { - "text": "\n[libdefaults]\n\trenew_lifetime = 7d" - } - } - ] - } - }" - - + common_configs: + configBlocks: + - id: das-ranger-policymgr + format: HADOOP_XML + content: + keyValues: + 'xasecure.policymgr.clientssl.truststore': '/path_to_ca_cert/cacerts' + application_configs: + das-webapp: + configBlocks: + - id: hive-kerberos-config + format: TEXT + content: + text: "\n[libdefaults]\n\trenew_lifetime = 7d" + # Delete Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: cluster_id: "example-cluster-id" @@ -252,60 +232,59 @@ RETURN = r''' --- -virtual_warehouses: - description: The information about the named CDW Virtual Warehouses. - type: list - returned: always - elements: complex +virtual_warehouse: + description: The details about the CDP Data Warehouse Virtual Warehouse. + type: dict suboptions: - vws: + id: + description: The identifier of the Virtual Warehouse. + returned: always + type: str + name: + description: The name of the Virtual Warehouse. + returned: always + type: str + vwType: + description: The Virtual Warehouse type. + returned: always + type: str + dbcId: + description: The Database Catalog ID associated with the Virtual Warehouse. + returned: always + type: str + creationDate: + description: The creation time of the Virtual Warehouse in UTC. + returned: always + type: str + status: + description: The status of the Virtual Warehouse. + returned: always + type: str + creator: + description: The CRN of the Virtual Warehouse creator. + returned: always type: dict suboptions: - id: - description: Id of the Virtual Warehouse created. - returned: always + crn: type: str - name: - description: Name of the Virtual Warehouse created. + description: The creator's Actor CRN returned: always + email: type: str - vwType: - description: Virtual Warehouse type. - returned: always + description: Email address (for users) + returned: when supported + workloadUsername: type: str - dbcId: - description: Database Catalog ID against which Virtual Warehouse was created. - returned: always - type: str - creationDate: - description: The creation time of the cluster in UTC. - returned: always + description: Username (for users) + returned: when supported + machineUsername: type: str - status: - description: The status of the Virtual Warehouse. - returned: always - type: str - creator: - description: The CRN of the cluster creator. - returned: always - type: dict - suboptions: - crn: - type: str - description: Actor CRN - email: - type: str - description: Email address for users - workloadUsername: - type: str - description: Username for users - machineUsername: - type: str - description: Username for machine users - tags: - description: Custom tags that were used to create this Virtual Warehouse. - returned: always - type: dict + description: Username (for machine users) + returned: when supported + tags: + description: Custom tags applied to the Virtual Warehouse. + returned: always + type: dict sdk_out: description: Returns the captured CDP SDK log. returned: when supported From 1a4409ea9fcb1e2cd56676050537a96c42b8fae4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:41 -0400 Subject: [PATCH 11/23] Add modules to documentation Signed-off-by: Webster Mudge --- docsrc/index.rst | 2 ++ plugins/README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docsrc/index.rst b/docsrc/index.rst index cdac0aee..054e0938 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -19,6 +19,8 @@ cloudera.cloud Ansible Collection df_info dw_cluster dw_cluster_info + dw_database_catalog + dw_virtual_warehouse env env_auth env_auth_info diff --git a/plugins/README.md b/plugins/README.md index 56867418..d8da4552 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -20,6 +20,8 @@ modules employ the underlying SDK contained within the `cdpy` Python package. | [df_info](./modules/df_info.py) | Gather information about CDP DataFlow services | | [dw_cluster](./modules/dw_cluster.py) | Create, manage, and destroy CDP Data Warehouse experiences | | [dw_cluster_info](./modules/dw_cluster_info.py) | Gather information about CDP Data Warehouse experiences | +| [dw_database_catalog](./modules/dw_database_catalog.py) | Create, manage, and destroy CDP Data Warehouse Data Catalogs | +| [dw_virtual_warehouse](./modules/dw_virtual_warehouse.py) | Create, manage, and destroy CDP Data Warehouse Virtual Warehouses | | [env](./modules/env.py) | Create, manage, and destroy CDP Environments | | [env_auth](./modules/env_auth.py) | Set authentication details for CDP Environments | | [env_auth_info](./modules/env_auth_info.py) | Gather information about CDP Environment authentication details | From 5c19a601b456d34d17eb581d5c24cc520cd2f121 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:41 -0400 Subject: [PATCH 12/23] Add missing descriptions to ConfigBlocks Signed-off-by: Webster Mudge --- plugins/modules/dw_virtual_warehouse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index 8b108ad4..e9143189 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -81,7 +81,8 @@ description: Configurations that are applied to every application in the Virtual Warehouse service. type: dict suboptions: - configBlocks: List of I(ConfigBlocks) for the application. + configBlocks: + description: List of I(ConfigBlocks) for the application. type: list elements: dict suboptions: @@ -114,7 +115,8 @@ description: The application name or identifier. type: dict suboptions: - configBlocks: List of ConfigBlocks for the application. + configBlocks: + description: List of I(ConfigBlocks) for the specified application. type: list required: False elements: dict From 3fdcfd89c10bf7279f8374667e7419a51f7f3b84 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:41 -0400 Subject: [PATCH 13/23] Tweak fail message Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index 9826d774..384054d2 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -251,7 +251,7 @@ def process(self): elif len(listing) == 0: self.target = None else: - self.module.fail_json(msg="Received multiple (i.e. ambiguious) DW Clusters in Environment %s" % self.env) + self.module.fail_json(msg="Received multiple (i.e. ambiguous) DW Clusters in Environment %s" % self.env) else: self.target = None From 1df347f9f6ab98580149f170286c583c3771ed94 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:41 -0400 Subject: [PATCH 14/23] Update module spec, API doc, and adjust logic flow for singular return value Signed-off-by: Webster Mudge --- plugins/modules/dw_virtual_warehouse.py | 116 ++++++++++++------------ 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index e9143189..e049c219 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -42,14 +42,14 @@ type: str cluster_id: description: - - The identifier of the parent DW Cluster of the Virtual Warehouse. + - The identifier of the parent Data Warehouse Cluster of the Virtual Warehouse. type: str required: True dbc_id: description: - The identifier of the parent Database Catalog attached to the Virtual Warehouse. + - Required if C(state=present) type: str - required: True type: description: - The type of Virtual Warehouse to be created. @@ -94,6 +94,14 @@ format: description: Format of the ConfigBlock. type: str + choices: + - HADOOP_XML + - PROPERTIES + - TEXT + - JSON + - BINARY + - ENV + - FLAGFILE content: description: Contents of the ConfigBlock. type: dict @@ -147,7 +155,7 @@ type: list elements: str enable_sso: - description: Flag to enable SSO for the Virtual Warehouse. + description: Flag to enable Single Sign-On (SSO) for the Virtual Warehouse. type: bool default: False tags: @@ -190,7 +198,7 @@ EXAMPLES = r''' # Note: These examples do not set authentication details. -# Create Virtual Warehouse +# Create a Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: cluster_id: example-cluster-id name: example-virtual-warehouse @@ -199,13 +207,14 @@ autoscaling_min_nodes: 3 autoscaling_max_nodes: 19 tags: - tag-key: "tag-value" + some_key: "some value" enable_sso: true - ldap_groups: ['group1','group2','group3'] - + ldap_groups: ['group1', 'group2', 'group3'] + +# Create a Virtual Warehouse with configurations - cloudera.cloud.dw_virtual_warehouse: - cluster_id: "example-cluster-id" - name: "example-virtual-warehouse" + cluster_id: example-cluster-id + name: example-virtual-warehouse type: "hive" template: "xsmall" enable_sso: true @@ -225,10 +234,10 @@ content: text: "\n[libdefaults]\n\trenew_lifetime = 7d" -# Delete Virtual Warehouse +# Delete a Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: - cluster_id: "example-cluster-id" - id: "example-virtual-warehouse-id" + cluster_id: example-cluster-id + id: example-virtual-warehouse-id state: absent ''' @@ -323,7 +332,7 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.virtual_warehouses = [] + self.virtual_warehouse = {} # Initialize internal values self.target = None @@ -339,18 +348,19 @@ def process(self): for vw in vws: if self.name is not None and vw['name'] == self.name: self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw['id']) - elif self.id is not None and vw['id'] == self.id: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) - # If Virtual Warehouse exists + else: + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) + if self.target is not None: + # Begin Virtual Warehouse Exists if self.state == 'absent': if self.module.check_mode: - self.virtual_warehouses.append(self.target) + self.virtual_warehouse = self.target else: + # Begin Drop if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: self.module.warn( - "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target[ - 'status']) + "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) self.changed = True @@ -362,32 +372,29 @@ def process(self): ) else: self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) - self.virtual_warehouses.append(self.target) - # Drop Done + self.virtual_warehouse = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) + # End Drop elif self.state == 'present': # Begin Config check - self.module.warn("DW Virtual Warehouse already present and config validation is not implemented") + self.module.warn("DW Virtual Warehouse already present and reconciliation is not implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) - self.virtual_warehouses.append(self.target) - # End Config check + self.virtual_warehouse = self.target + # End Config check else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) - # End handling Virtual Warehouse exists + # End Virtual Warehouse Exists else: - # Begin handling Virtual Warehouse not found + # Begin Virtual Warehouse Not Found if self.state == 'absent': self.module.warn( "DW Virtual Warehouse %s already absent in Cluster %s" % (self.name, self.cluster_id)) elif self.state == 'present': - if self.module.check_mode: - pass - else: + if not self.module.check_mode: vw_id = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, dbc_id=self.dbc_id, vw_type=self.type, name=self.name, template=self.template, @@ -399,17 +406,16 @@ def process(self): tags=self.tags) self.changed = True if self.wait: - self.target = self.cdpy.sdk.wait_for_state( + self.virtual_warehouse = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, params=dict(cluster_id=self.cluster_id, vw_id=vw_id), state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) else: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw_id) - self.virtual_warehouses.append(self.target) + self.virtual_warehouse = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw_id) else: self.module.fail_json(msg="State %s is not valid for this module" % self.state) - + # End Virtual Warehouse Not Found def main(): module = AnsibleModule( @@ -419,32 +425,20 @@ def main(): dbc_id=dict(type='str'), type = dict(type='str'), name = dict(type='str'), - template=dict(type='str'), + template=dict(type='str', choices=['xsmall', 'small', 'medium', 'large']), autoscaling_min_nodes=dict(type='int'), autoscaling_max_nodes=dict(type='int'), - common_configs=dict(type='dict', - options=dict( - configBlocks = dict( - type='list', - elements='dict', - options=dict( - dict( - id=dict(type='str'), - format=dict(type='str', choices=['HADOOP_XML', 'PROPERTIES', 'TEXT', 'JSON', 'BINARY', 'ENV', 'FLAGFILE']), - content=dict(type='dict', - options=dict( - dict( - keyValues=dict(type='dict'), - text=dict(type='str'), - json=dict(type='json') - ) - ) - ) - ) - ) - ) - ) - ), + common_configs=dict(type='dict', options=dict( + configBlocks = dict(type='list', elements='dict', options=dict( + id=dict(type='str'), + format=dict(type='str', choices=['HADOOP_XML', 'PROPERTIES', 'TEXT', 'JSON', 'BINARY', 'ENV', 'FLAGFILE']), + content=dict(type='dict', options=dict( + keyValues=dict(type='dict'), + text=dict(type='str'), + json=dict(type='json') + )) + )) + )), application_configs=dict(type='dict'), ldap_groups=dict(type='list'), enable_sso=dict(type='bool'), @@ -454,11 +448,15 @@ def main(): delay = dict(type='int', aliases=['polling_delay'], default=15), timeout = dict(type='int', aliases=['polling_timeout'], default=3600) ), + required_if=[ + ['state', 'absent', ['id']], + ['state', 'present', ['dbc_id', 'type', 'name'], True] + ], supports_check_mode=True ) result = DwVw(module) - output = dict(changed=result.changed, virtual_warehouses=result.virtual_warehouses) + output = dict(changed=result.changed, virtual_warehouse=result.virtual_warehouse) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) From f2de47f3b817efce174f284165c73d85d2dea6d6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:42 -0400 Subject: [PATCH 15/23] Add dw_database_catalog_info module Signed-off-by: Webster Mudge --- docsrc/index.rst | 1 + plugins/README.md | 1 + plugins/modules/dw_database_catalog_info.py | 157 ++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 plugins/modules/dw_database_catalog_info.py diff --git a/docsrc/index.rst b/docsrc/index.rst index 054e0938..6c4b2458 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -20,6 +20,7 @@ cloudera.cloud Ansible Collection dw_cluster dw_cluster_info dw_database_catalog + dw_database_catalog_info dw_virtual_warehouse env env_auth diff --git a/plugins/README.md b/plugins/README.md index d8da4552..a1243141 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -21,6 +21,7 @@ modules employ the underlying SDK contained within the `cdpy` Python package. | [dw_cluster](./modules/dw_cluster.py) | Create, manage, and destroy CDP Data Warehouse experiences | | [dw_cluster_info](./modules/dw_cluster_info.py) | Gather information about CDP Data Warehouse experiences | | [dw_database_catalog](./modules/dw_database_catalog.py) | Create, manage, and destroy CDP Data Warehouse Data Catalogs | +| [dw_database_catalog_info](./modules/dw_database_catalog_info.py) | Gather information about CDP Data Warehouse Data Catalogs | | [dw_virtual_warehouse](./modules/dw_virtual_warehouse.py) | Create, manage, and destroy CDP Data Warehouse Virtual Warehouses | | [env](./modules/env.py) | Create, manage, and destroy CDP Environments | | [env_auth](./modules/env_auth.py) | Set authentication details for CDP Environments | diff --git a/plugins/modules/dw_database_catalog_info.py b/plugins/modules/dw_database_catalog_info.py new file mode 100644 index 00000000..b8deed1a --- /dev/null +++ b/plugins/modules/dw_database_catalog_info.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: dw_database_catalog_info +short_description: Gather information about CDP Data Warehouse Database Catalogs +description: + - Gather information about CDP Data Warehouse Database Catalogs +author: + - "Webster Mudge (@wmudge)" + - "Dan Chaffelson (@chaffelson)" + - "Saravanan Raju (@raju-saravanan)" +requirements: + - cdpy +options: + id: + description: + - The identifier of the Database Catalog. + - If undefined, will return a list of all Database Catalogs in the Cluster. + - Exclusive with I(name). + type: str + cluster_id: + description: + - The identifier of the parent Cluster of the Database Catalog or Catalogs. + type: str + required: True + name: + description: + - The name of the Database Catalog. + - If undefined, will return a list of all Database Catalogs in the Cluster. + - Exclusive with I(id). + type: str +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Get a single Database Catalog +- cloudera.cloud.dw_database_catalog_info: + name: example-database-catalog-name + cluster_id: example-cluster-id + +# Get all Database Catalogs within a Cluster +- cloudera.cloud.dw_database_catalog_info: + cluster_id: example-cluster-id +''' + +RETURN = r''' +--- +database_catalogs: + description: Details about the Database Catalogs. + returned: always + type: list + elements: dict + contains: + id: + description: The identifier of the Database Catalog. + returned: always + type: str + name: + description: The name of the Database Catalog. + returned: always + type: str + status: + description: The status of the Database Catalog. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DwDbcInfo(CdpModule): + def __init__(self, module): + super(DwDbcInfo, self).__init__(module) + + # Set variables + self.id = self._get_param('id') + self.cluster_id = self._get_param('cluster_id') + self.name = self._get_param('name') + + # Initialize return values + self.database_catalogs = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + if self.id is not None: + target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) + if target is not None: + self.database_catalogs.append(target) + else: + dbcs = self.cdpy.dw.list_dbcs(cluster_id=self.cluster_id) + if self.name is not None: + for dbc in dbcs: + if dbc['name'] == self.name: + self.database_catalogs.append(self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc['id'])) + else: + self.database_catalogs = dbcs + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + id=dict(type='str'), + cluster_id=dict(required=True, type='str'), + name = dict(type='str'), + ), + mutually_exclusive=[['id', 'name']], + supports_check_mode=True + ) + + result = DwDbcInfo(module) + output = dict(changed=False, database_catalogs=result.database_catalogs) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() From c29f82e1da06a205d1bf88e0e71e274f87777c1d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:42 -0400 Subject: [PATCH 16/23] Fix debug log bug Issue violations as discrete array in returned error Add check for verbosity or debug to SDK warnings handler Signed-off-by: Webster Mudge --- plugins/module_utils/cdp_common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/module_utils/cdp_common.py b/plugins/module_utils/cdp_common.py index 18e1205d..b1d43a2a 100644 --- a/plugins/module_utils/cdp_common.py +++ b/plugins/module_utils/cdp_common.py @@ -41,7 +41,7 @@ def process_debug(cls, f): def _impl(self, *args, **kwargs): result = f(self, *args, **kwargs) if self.debug: - self.log_out = self.cdpy.get_log() + self.log_out = self.cdpy.sdk.get_log() self.log_lines.append(self.log_out.splitlines()) return result @@ -73,11 +73,12 @@ def _get_param(self, param, default=None): def _cdp_module_throw_error(self, error: 'CdpError'): """Error handler for CDPy SDK""" - self.module.fail_json(msg=str(error.message), error=str(error.__dict__)) + self.module.fail_json(msg=str(error.message), error=str(error.__dict__), violations=error.violations) def _cdp_module_throw_warning(self, warning: 'CdpWarning'): """Warning handler for CDPy SDK""" - self.module.warn(warning.message) + if self.module._debug or self.module._verbosity >= 2: + self.module.warn(warning.message) @staticmethod def argument_spec(**spec): From 5f2a37cc4054afb33d43b427c0d6af697ac89395 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:42 -0400 Subject: [PATCH 17/23] Update log messaging Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index 384054d2..1bf54082 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -251,7 +251,7 @@ def process(self): elif len(listing) == 0: self.target = None else: - self.module.fail_json(msg="Received multiple (i.e. ambiguous) DW Clusters in Environment %s" % self.env) + self.module.fail_json(msg="Received multiple (i.e. ambiguous) Clusters in Environment %s" % self.env) else: self.target = None @@ -264,7 +264,7 @@ def process(self): else: self.changed = True if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn("DW Cluster is not in a valid state for Delete operations: %s" % self.target['status']) + self.module.warn("Cluster is not in a valid state for Delete operations: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_cluster(cluster_id=self.name, force=self.force) @@ -280,7 +280,7 @@ def process(self): # End Delete elif self.state == 'present': # Begin Config Check - self.module.warn("DW Cluster is already present and reconciliation is not yet implemented") + self.module.warn("Cluster is already present and reconciliation is not yet implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_cluster, @@ -295,7 +295,7 @@ def process(self): else: # Begin Cluster Not Found if self.state == 'absent': - self.module.warn("DW Cluster %s already absent in Environment %s" % (self.name, self.env)) + self.module.warn("Cluster %s already absent in Environment %s" % (self.name, self.env)) elif self.state == 'present': if not self.module.check_mode: # Begin Cluster Creation From eff634ff4c67ccbe5730577656b786b99a877ab9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:42 -0400 Subject: [PATCH 18/23] Update module variable requirements Update deletion states and process Update log messaging Signed-off-by: Webster Mudge --- plugins/modules/dw_virtual_warehouse.py | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index e049c219..59ea5416 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -167,7 +167,7 @@ default: present choices: - present - - absent + - absent wait: description: - Flag to enable internal polling to wait for the Virtual Warehouse to achieve the declared state. @@ -359,29 +359,30 @@ def process(self): else: # Begin Drop if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn( - "DW Virtual Warehouse not in valid state for Delete operation: %s" % self.target['status']) + self.module.fail_json(msg="Virtual Warehouse not in valid state for Delete operation: %s" % + self.target['status']) else: _ = self.cdpy.dw.delete_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) self.changed = True - if self.wait: - self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_vw, - params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), - field=None, delay=self.delay, timeout=self.timeout - ) - else: - self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync - self.virtual_warehouse = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_vw, + params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), + field=None, delay=self.delay, timeout=self.timeout + ) + else: + self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync + self.virtual_warehouse = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.target['id']) # End Drop elif self.state == 'present': # Begin Config check - self.module.warn("DW Virtual Warehouse already present and reconciliation is not implemented") - if self.wait: + self.module.warn("Virtual Warehouse already present and reconciliation is not yet implemented") + if self.wait and not self.module.check_mode: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_vw, params=dict(cluster_id=self.cluster_id, vw_id=self.target['id']), - state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout + state=self.cdpy.sdk.STARTED_STATES + self.cdpy.sdk.STOPPED_STATES, delay=self.delay, + timeout=self.timeout ) self.virtual_warehouse = self.target # End Config check @@ -391,8 +392,7 @@ def process(self): else: # Begin Virtual Warehouse Not Found if self.state == 'absent': - self.module.warn( - "DW Virtual Warehouse %s already absent in Cluster %s" % (self.name, self.cluster_id)) + self.module.warn("Virtual Warehouse is already absent in Cluster %s" % self.cluster_id) elif self.state == 'present': if not self.module.check_mode: vw_id = self.cdpy.dw.create_vw(cluster_id=self.cluster_id, @@ -441,7 +441,7 @@ def main(): )), application_configs=dict(type='dict'), ldap_groups=dict(type='list'), - enable_sso=dict(type='bool'), + enable_sso=dict(type='bool', default=False), tags=dict(type='dict'), state=dict(type='str', choices=['present', 'absent'], default='present'), wait = dict(type='bool', default=True), @@ -450,7 +450,7 @@ def main(): ), required_if=[ ['state', 'absent', ['id']], - ['state', 'present', ['dbc_id', 'type', 'name'], True] + ['state', 'present', ['dbc_id', 'type', 'name']] ], supports_check_mode=True ) From 68a96ba4b1e05852e15b59ad47c2cf56a97aff0a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:42 -0400 Subject: [PATCH 19/23] Update deletion states and process Update module parameter requirements Update log messaging Signed-off-by: Webster Mudge --- plugins/modules/dw_database_catalog.py | 31 +++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/plugins/modules/dw_database_catalog.py b/plugins/modules/dw_database_catalog.py index af74564b..3ecb9d43 100644 --- a/plugins/modules/dw_database_catalog.py +++ b/plugins/modules/dw_database_catalog.py @@ -156,7 +156,7 @@ def process(self): if self.id is None: dbcs = self.cdpy.dw.list_dbcs(cluster_id=self.cluster_id) for dbc in dbcs: - if self.name is not None and dbc['name'] == self.name: + if dbc['name'] == self.name: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc['id']) else: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) @@ -169,24 +169,24 @@ def process(self): else: # Begin Drop if self.target['status'] not in self.cdpy.sdk.REMOVABLE_STATES: - self.module.warn( - "DW Database Catalog is not in a valid state for Delete operations: %s" % self.target['status']) + self.module.fail_json(msg= + "Database Catalog is not in a valid state for Delete operations: %s" % self.target['status']) else: _ = self.cdpy.dw.delete_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) self.changed = True - if self.wait: - self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.dw.describe_dbc, - params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), - field=None, delay=self.delay, timeout=self.timeout - ) - else: - self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync - self.database_catalog = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) + if self.wait: + self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.dw.describe_dbc, + params=dict(cluster_id=self.cluster_id, dbc_id=self.target['id']), + field=None, delay=self.delay, timeout=self.timeout + ) + else: + self.cdpy.sdk.sleep(self.delay) # Wait for consistency sync + self.database_catalog = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.target['id']) # End Drop elif self.state == 'present': # Begin Config Check - self.module.warn("DW Database Catalog already present and reconciliation is not yet implemented") + self.module.warn("Database Catalog already present and reconciliation is not yet implemented") if self.wait: self.target = self.cdpy.sdk.wait_for_state( describe_func=self.cdpy.dw.describe_dbc, @@ -201,7 +201,7 @@ def process(self): else: # Begin Database Catalog Not Found if self.state == 'absent': - self.module.warn("DW Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) + self.module.warn("Database Catalog %s already absent in Cluster %s" % (self.name, self.cluster_id)) elif self.state == 'present': if not self.module.check_mode: dbc_id = self.cdpy.dw.create_dbc(cluster_id=self.cluster_id, name=self.name, @@ -233,8 +233,9 @@ def main(): ), required_if=[ ['state', 'present', ['name']], - ['state', 'absent', ['name', 'id']], + ['state', 'absent', ['name', 'id'], True], ], + mutually_exclusive=[['name', 'id']], supports_check_mode=True ) From 73dbad479b590a73a72396811546822ce317b536 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:43 -0400 Subject: [PATCH 20/23] Refactor scope to only return cluster details Remove 'name' parameter Refactor remaining parameters for aliases and requirements Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster_info.py | 125 +++++++++++++++-------------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/plugins/modules/dw_cluster_info.py b/plugins/modules/dw_cluster_info.py index a9c0fab9..3fe6d9de 100644 --- a/plugins/modules/dw_cluster_info.py +++ b/plugins/modules/dw_cluster_info.py @@ -34,20 +34,18 @@ requirements: - cdpy options: - id: + cluster_id: description: - - If a name is provided, that Data Warehouse Cluster will be described. - - environment must be provided if using name to retrieve a Cluster + - The identifier of the Data Warehouse Cluster. + - Mutually exclusive with I(environment). type: str - required: False aliases: - - name + - id environment: description: - - The name of the Environment in which to find and describe the Data Warehouse Clusters. - - Required with name to retrieve a Cluster + - The name or CRN of the Environment in which to find and describe Data Warehouse Clusters. + - Mutually exclusive with I(cluster_id). type: str - required: False aliases: - env extends_documentation_fragment: @@ -58,67 +56,71 @@ EXAMPLES = r''' # Note: These examples do not set authentication details. -# List basic information about all Data Warehouse Clusters +# List information about all Data Warehouse Clusters - cloudera.cloud.dw_cluster_info: -# Gather detailed information about a named Cluster +# Gather information about all Data Warehouse Clusters within an Environment - cloudera.cloud.dw_cluster_info: - name: example-cluster env: example-environment + +# Gather information about an identified Cluster +- cloudera.cloud.dw_cluster_info: + id: env-xyzabc ''' RETURN = r''' --- clusters: description: The information about the named Cluster or Clusters - type: list returned: always - elements: complex + type: list + elements: dict contains: - cluster: + id: + description: The cluster identifier. + returned: always + type: str + environmentCrn: + description: The CRN of the cluster's Environment + returned: always + type: str + crn: + description: The cluster's CRN. + returned: always + type: str + creationDate: + description: The creation timestamp of the cluster in UTC. + returned: always + type: str + status: + description: The status of the cluster + returned: always + type: str + creator: + description: The cluster creator details. + returned: always type: dict contains: - name: - description: The name of the cluster. - returned: always - type: str - environmentCrn: - description: The crn of the cluster's environment. - returned: always - type: str crn: - description: The cluster's crn. - returned: always + description: The Actor CRN. type: str - creationDate: - description: The creation time of the cluster in UTC. returned: always + email: + description: Email address (users). type: str - status: - description: The status of the Cluster - returned: always + returned: when supported + workloadUsername: + description: Username (users). type: str - creator: - description: The CRN of the cluster creator. - returned: always - type: dict - contains: - crn: - type: str - description: Actor CRN - email: - type: str - description: Email address for users - workloadUsername: - type: str - description: Username for users - machineUsername: - type: str - description: Username for machine users - cloudPlatform: - description: The cloud platform of the environment that was used to create this cluster - returned: always + returned: when supported + machineUsername: + description: Username (machine users). type: str + returned: when supported + cloudPlatform: + description: The cloud platform of the environment that was used to create this cluster. + returned: always + type: str sdk_out: description: Returns the captured CDP SDK log. returned: when supported @@ -136,8 +138,8 @@ def __init__(self, module): super(DwClusterInfo, self).__init__(module) # Set variables - self.id = self._get_param('name') - self.env = self._get_param('env') + self.cluster_id = self._get_param('cluster_id') + self.environment = self._get_param('environment') # Initialize return values self.clusters = [] @@ -147,24 +149,27 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.id is not None: - cluster_single = self.cdpy.dw.describe_cluster(name=self.id) + if self.cluster_id is not None: + cluster_single = self.cdpy.dw.describe_cluster(self.cluster_id) if cluster_single is not None: self.clusters.append(cluster_single) - if self.env is not None: - env_crn = self.cdpy.environments.resolve_environment_crn(self.env) - if env_crn is not None: - self.clusters = self.cdpy.dw.gather_clusters(env_crn) + elif self.environment is not None: + env_crn = self.cdpy.environments.resolve_environment_crn(self.environment) + if env_crn: + self.clusters = self.cdpy.dw.list_clusters(env_crn=env_crn) else: - self.clusters = self.cdpy.dw.gather_clusters() + self.clusters = self.cdpy.dw.list_clusters() def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(required=False, type='str', aliases=['name']), - env=dict(required=False, type='str', aliases=['environment']) + cluster_id=dict(type='str', aliases=['id']), + environment=dict(type='str', aliases=['env']) ), + mutually_exclusive=[ + ['cluster_id', 'environment'] + ], supports_check_mode=True ) From e4a940431e7feca6cf984407a4487a9c9fd84764 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:43 -0400 Subject: [PATCH 21/23] Add dw_virtual_warehouse_info module Signed-off-by: Webster Mudge --- docsrc/index.rst | 1 + plugins/README.md | 1 + plugins/modules/dw_virtual_warehouse_info.py | 239 +++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 plugins/modules/dw_virtual_warehouse_info.py diff --git a/docsrc/index.rst b/docsrc/index.rst index 6c4b2458..0065294d 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -22,6 +22,7 @@ cloudera.cloud Ansible Collection dw_database_catalog dw_database_catalog_info dw_virtual_warehouse + dw_virtual_warehouse_info env env_auth env_auth_info diff --git a/plugins/README.md b/plugins/README.md index a1243141..09e8ab8a 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -23,6 +23,7 @@ modules employ the underlying SDK contained within the `cdpy` Python package. | [dw_database_catalog](./modules/dw_database_catalog.py) | Create, manage, and destroy CDP Data Warehouse Data Catalogs | | [dw_database_catalog_info](./modules/dw_database_catalog_info.py) | Gather information about CDP Data Warehouse Data Catalogs | | [dw_virtual_warehouse](./modules/dw_virtual_warehouse.py) | Create, manage, and destroy CDP Data Warehouse Virtual Warehouses | +| [dw_virtual_warehouse_info](./modules/dw_virtual_warehouse_info.py) | Gather information about CDP Data Warehouse Virtual Warehouses | | [env](./modules/env.py) | Create, manage, and destroy CDP Environments | | [env_auth](./modules/env_auth.py) | Set authentication details for CDP Environments | | [env_auth_info](./modules/env_auth_info.py) | Gather information about CDP Environment authentication details | diff --git a/plugins/modules/dw_virtual_warehouse_info.py b/plugins/modules/dw_virtual_warehouse_info.py new file mode 100644 index 00000000..6351f3b5 --- /dev/null +++ b/plugins/modules/dw_virtual_warehouse_info.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: dw_virtual_warehouse_info +short_description: Gather information about CDP Data Warehouse Virtual Warehouses +description: + - Gather information about CDP Virtual Warehouses +author: + - "Webster Mudge (@wmudge)" + - "Dan Chaffelson (@chaffelson)" + - "Saravanan Raju (@raju-saravanan)" +requirements: + - cdpy +options: + id: + description: + - The identifier of the Virtual Warehouse. + - Requires I(cluster_id). + - Mutually exclusive with I(name) and I(dbc_id). + type: str + aliases: + - vw_id + cluster_id: + description: + - The identifier of the parent Data Warehouse Cluster of the Virtual Warehouse(s). + type: str + dbc_id: + description: + - The identifier of the parent Database Catalog attached to the Virtual Warehouse(s). + - Requires I(cluster_id). + - Mutally exclusive with I(id) and I(name). + type: str + name: + description: + - The name of the Virtual Warehouse. + - Requires I(cluster_id). + - Mutually exclusive with I(id) and I(dbc_id). + type: str + delay: + description: + - The internal polling interval (in seconds) while the module waits for the Virtual Warehouse to achieve the declared + state. + type: int + default: 15 + aliases: + - polling_delay + timeout: + description: + - The internal polling timeout (in seconds) while the module waits for the Virtual Warehouse to achieve the declared + state. + type: int + default: 3600 + aliases: + - polling_timeout +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List all Virtual Warehouses in a Cluster +- cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: example-cluster-id + +# List all Virtual Warehouses associated with a Data Catalog +- cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: example-cluster-id + dbc_id: example-data-catalog-id + +# Describe a Virtual Warehouse by ID +- cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: example-cluster-id + id: example-virtual-warehouse-id + +# Describe a Virtual Warehouse by name +- cloudera.cloud.dw_virtual_warehouse_info: + cluster_id: example-cluster-id + name: example-virtual-warehouse +''' + +RETURN = r''' +--- +virtual_warehouses: + description: The details about the CDP Data Warehouse Virtual Warehouse(s). + type: list + elements: dict + contains: + id: + description: The identifier of the Virtual Warehouse. + returned: always + type: str + name: + description: The name of the Virtual Warehouse. + returned: always + type: str + vwType: + description: The Virtual Warehouse type. + returned: always + type: str + dbcId: + description: The Database Catalog ID associated with the Virtual Warehouse. + returned: always + type: str + creationDate: + description: The creation time of the Virtual Warehouse in UTC. + returned: always + type: str + status: + description: The status of the Virtual Warehouse. + returned: always + type: str + creator: + description: Details about the Virtual Warehouse creator. + returned: always + type: dict + suboptions: + crn: + description: The creator's Actor CRN. + type: str + returned: always + email: + description: Email address (for users). + type: str + returned: when supported + workloadUsername: + description: Username (for users). + type: str + returned: when supported + machineUsername: + description: Username (for machine users). + type: str + returned: when supported + tags: + description: Custom tags applied to the Virtual Warehouse. + returned: always + type: dict +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DwVwInfo(CdpModule): + def __init__(self, module): + super(DwVwInfo, self).__init__(module) + + # Set variables + self.id = self._get_param('id') + self.cluster_id = self._get_param('cluster_id') + self.dbc_id = self._get_param('dbc_id') + self.type = self._get_param('type') + self.name = self._get_param('name') + self.delay = self._get_param('delay') + self.timeout = self._get_param('timeout') + + # Initialize return values + self.virtual_warehouses = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + if self.id is not None: + target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) + if target is not None: + self.virtual_warehouses.append(target) + else: + vws = self.cdpy.dw.list_vws(cluster_id=self.cluster_id) + if self.name is not None: + for vw in vws: + if vw['name'] == self.name: + self.virtual_warehouses.append( + self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw['id']) + ) + elif self.dbc_id is not None: + self.virtual_warehouses =[v for v in vws if v['dbcId'] == self.dbc_id] + else: + self.virtual_warehouses = vws + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + id=dict(type='str', aliases=['vw_id']), + cluster_id=dict(required=True, type='str'), + dbc_id=dict(type='str'), + name=dict(type='str'), + delay=dict(type='int', aliases=['polling_delay'], default=15), + timeout=dict(type='int', aliases=['polling_timeout'], default=3600) + ), + mutually_exclusive=[ + ['id', 'name', 'dbc_id'] + ], + supports_check_mode=True + ) + + result = DwVwInfo(module) + output = dict(changed=False, virtual_warehouses=result.virtual_warehouses) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() From 02749f8553ebdc1e56ae7185fd6597b55487f7ea Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:14:43 -0400 Subject: [PATCH 22/23] Update parameter naming, aliases, and requirements Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster.py | 54 ++++++++++++++----------- plugins/modules/dw_virtual_warehouse.py | 16 ++++---- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index 1bf54082..f564d940 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -35,17 +35,19 @@ requirements: - cdpy options: - name: + cluster_id: description: - The identifier of the Data Warehouse Cluster. - - Required if C(state=absent) and C(env) is not specified. + - Required if I(state=absent) and I(env) is not specified. type: str aliases: - id + - name env: description: - The name of the target environment. - - Required if C(state=present) or C(state=absent) and C(id) is not specified. + - Required if I(state=present). + - Required if I(state=absent) and I(cluster_id) is not specified. type: str aliases: - environment @@ -63,24 +65,24 @@ aws_public_subnets: description: - List of zero or more Public AWS Subnet IDs used for deployment. - - Required if C(state=present) and the C(env) is deployed to AWS. + - Required if I(state=present) and the I(env) is deployed to AWS. type: list elements: str aws_private_subnets: description: - List of zero or more Private AWS Subnet IDs used for deployment. - - Required if C(state=present) and the C(env) is deployed to AWS. + - Required if I(state=present) and the I(env) is deployed to AWS. type: list elements: str az_subnet: description: - The Azure Subnet Name. - - Required if C(state=present) and the C(env) is deployed to Azure. + - Required if I(state=present) and the I(env) is deployed to Azure. type: str az_enable_az: description: - Flag to enable Availability Zone mode. - - Required if C(state=present) and the C(env) is deployed to Azure. + - Required if I(state=present) and the I(env) is deployed to Azure. type: bool state: description: The state of the Data Warehouse Cluster @@ -154,12 +156,12 @@ description: Details for the Data Warehouse cluster type: dict contains: - name: - description: The name of the cluster. + id: + description: The cluster identifier. returned: always type: str environmentCrn: - description: The crn of the cluster's environment. + description: The CRN of the cluster's Environment returned: always type: str crn: @@ -167,32 +169,36 @@ returned: always type: str creationDate: - description: The creation time of the cluster in UTC. + description: The creation timestamp of the cluster in UTC. returned: always type: str status: - description: The status of the cluster. + description: The status of the cluster returned: always type: str creator: - description: The details regarding the cluster creator. + description: The cluster creator details. returned: always type: dict contains: crn: + description: The Actor CRN. type: str - description: The creator's CRN + returned: always email: + description: Email address (users). type: str - description: Email address + returned: when supported workloadUsername: + description: Username (users). type: str - description: Username + returned: when supported machineUsername: + description: Username (machine users). type: str - description: Username for machine users + returned: when supported cloudPlatform: - description: The cloud platform of the environment housing the Data Warehouse cluster + description: The cloud platform of the environment that was used to create this cluster. returned: always type: str sdk_out: @@ -325,14 +331,14 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - name=dict(type='str', aliases=['id']), + cluster_id=dict(type='str', aliases=['id', 'name']), env=dict(type='str', aliases=['environment', 'env_crn']), overlay=dict(type='bool', default=False), private_load_balancer=dict(type='bool', default=False), - az_subnet=dict(type='str', default=None), - az_enable_az=dict(type='bool', default=None), - aws_public_subnets=dict(type='list', default=None), - aws_private_subnets=dict(type='list', default=None), + az_subnet=dict(type='str'), + az_enable_az=dict(type='bool'), + aws_public_subnets=dict(type='list'), + aws_private_subnets=dict(type='list'), state=dict(type='str', choices=['present', 'absent'], default='present'), force=dict(type='bool', default=False), wait=dict(type='bool', default=True), @@ -344,7 +350,7 @@ def main(): ['aws_public_subnets', 'aws_private_subnets'] ], required_if=[ - ['state', 'absent', ['name', 'env']], + ['state', 'absent', ['cluster_id', 'env'], True], ['state', 'present', ['env']] ], supports_check_mode=True diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index 59ea5416..32ff46ef 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -40,6 +40,8 @@ - The identifier of the Virtual Warehouse. - Required if C(state=absent). type: str + aliases: + - vw_id cluster_id: description: - The identifier of the parent Data Warehouse Cluster of the Virtual Warehouse. @@ -246,7 +248,7 @@ virtual_warehouse: description: The details about the CDP Data Warehouse Virtual Warehouse. type: dict - suboptions: + contains: id: description: The identifier of the Virtual Warehouse. returned: always @@ -272,25 +274,25 @@ returned: always type: str creator: - description: The CRN of the Virtual Warehouse creator. + description: Details about the Virtual Warehouse creator. returned: always type: dict suboptions: crn: + description: The creator's Actor CRN. type: str - description: The creator's Actor CRN returned: always email: + description: Email address (for users). type: str - description: Email address (for users) returned: when supported workloadUsername: + description: Username (for users). type: str - description: Username (for users) returned: when supported machineUsername: + description: Username (for machine users). type: str - description: Username (for machine users) returned: when supported tags: description: Custom tags applied to the Virtual Warehouse. @@ -420,7 +422,7 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(type='str'), + id=dict(type='str', aliases=['vw_id']), cluster_id=dict(required=True, type='str'), dbc_id=dict(type='str'), type = dict(type='str'), From 1302819b66a5e371f4ef9da0c255a2fd521b5e68 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 16 Sep 2021 09:47:49 -0400 Subject: [PATCH 23/23] Clarify parameter names and aliases for DW modules Signed-off-by: Webster Mudge --- plugins/modules/dw_cluster.py | 2 +- plugins/modules/dw_cluster_info.py | 2 +- plugins/modules/dw_database_catalog.py | 20 +++++----- plugins/modules/dw_database_catalog_info.py | 18 +++++---- plugins/modules/dw_virtual_warehouse.py | 31 ++++++++------- plugins/modules/dw_virtual_warehouse_info.py | 41 +++++++++++--------- 6 files changed, 62 insertions(+), 52 deletions(-) diff --git a/plugins/modules/dw_cluster.py b/plugins/modules/dw_cluster.py index f564d940..df0b56b6 100644 --- a/plugins/modules/dw_cluster.py +++ b/plugins/modules/dw_cluster.py @@ -142,7 +142,7 @@ # Delete a Data Warehouse Cluster - cloudera.cloud.dw_cluster: state: absent - id: my-id + cluster_id: my-id # Delete the Data Warehouse Cluster within the Environment - cloudera.cloud.dw_cluster: diff --git a/plugins/modules/dw_cluster_info.py b/plugins/modules/dw_cluster_info.py index 3fe6d9de..7892b727 100644 --- a/plugins/modules/dw_cluster_info.py +++ b/plugins/modules/dw_cluster_info.py @@ -65,7 +65,7 @@ # Gather information about an identified Cluster - cloudera.cloud.dw_cluster_info: - id: env-xyzabc + cluster_id: env-xyzabc ''' RETURN = r''' diff --git a/plugins/modules/dw_database_catalog.py b/plugins/modules/dw_database_catalog.py index 3ecb9d43..168d654a 100644 --- a/plugins/modules/dw_database_catalog.py +++ b/plugins/modules/dw_database_catalog.py @@ -35,11 +35,13 @@ requirements: - cdpy options: - id: + catalog_id: description: - The identifier of the Database Catalog. - Required if C(state=absent). type: str + aliases: + - id cluster_id: description: - The identifier of the parent DW Cluster of the Database Catalog. @@ -91,7 +93,7 @@ # Delete Database Catalog - cloudera.cloud.dw_database_catalog: - id: example-database-id + catalog_id: example-database-id cluster_id: example-cluster-id state: absent ''' @@ -127,12 +129,12 @@ ''' -class DwDbc(CdpModule): +class DwDatabaseCatalog(CdpModule): def __init__(self, module): - super(DwDbc, self).__init__(module) + super(DwDatabaseCatalog, self).__init__(module) # Set variables - self.id = self._get_param('id') + self.catalog_id = self._get_param('catalog_id') self.cluster_id = self._get_param('cluster_id') self.name = self._get_param('name') self.load_demo_data = self._get_param('load_demo_data') @@ -153,13 +155,13 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.id is None: + if self.catalog_id is None: dbcs = self.cdpy.dw.list_dbcs(cluster_id=self.cluster_id) for dbc in dbcs: if dbc['name'] == self.name: self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=dbc['id']) else: - self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) + self.target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.catalog_id) if self.target is not None: # Begin Database Catalog Exists @@ -222,7 +224,7 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(type='str'), + catalog_id=dict(type='str', aliases=['catalog_id']), cluster_id=dict(required=True, type='str'), name = dict(type='str'), load_demo_data=dict(type='bool'), @@ -239,7 +241,7 @@ def main(): supports_check_mode=True ) - result = DwDbc(module) + result = DwDatabaseCatalog(module) output = dict(changed=result.changed, database_catalog=result.database_catalog) if result.debug: diff --git a/plugins/modules/dw_database_catalog_info.py b/plugins/modules/dw_database_catalog_info.py index b8deed1a..17ec07d6 100644 --- a/plugins/modules/dw_database_catalog_info.py +++ b/plugins/modules/dw_database_catalog_info.py @@ -35,12 +35,14 @@ requirements: - cdpy options: - id: + catalog_id: description: - The identifier of the Database Catalog. - If undefined, will return a list of all Database Catalogs in the Cluster. - Exclusive with I(name). type: str + aliases: + - id cluster_id: description: - The identifier of the parent Cluster of the Database Catalog or Catalogs. @@ -102,12 +104,12 @@ ''' -class DwDbcInfo(CdpModule): +class DwDatabaseCatalogInfo(CdpModule): def __init__(self, module): - super(DwDbcInfo, self).__init__(module) + super(DwDatabaseCatalogInfo, self).__init__(module) # Set variables - self.id = self._get_param('id') + self.catalog_id = self._get_param('catalog_id') self.cluster_id = self._get_param('cluster_id') self.name = self._get_param('name') @@ -119,8 +121,8 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.id is not None: - target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.id) + if self.catalog_id is not None: + target = self.cdpy.dw.describe_dbc(cluster_id=self.cluster_id, dbc_id=self.catalog_id) if target is not None: self.database_catalogs.append(target) else: @@ -136,7 +138,7 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(type='str'), + catalog_id=dict(type='str', aliases=['id']), cluster_id=dict(required=True, type='str'), name = dict(type='str'), ), @@ -144,7 +146,7 @@ def main(): supports_check_mode=True ) - result = DwDbcInfo(module) + result = DwDatabaseCatalogInfo(module) output = dict(changed=False, database_catalogs=result.database_catalogs) if result.debug: diff --git a/plugins/modules/dw_virtual_warehouse.py b/plugins/modules/dw_virtual_warehouse.py index 32ff46ef..02b4deeb 100644 --- a/plugins/modules/dw_virtual_warehouse.py +++ b/plugins/modules/dw_virtual_warehouse.py @@ -35,23 +35,26 @@ requirements: - cdpy options: - id: + warehouse_id: description: - The identifier of the Virtual Warehouse. - Required if C(state=absent). type: str aliases: - vw_id + - id cluster_id: description: - The identifier of the parent Data Warehouse Cluster of the Virtual Warehouse. type: str required: True - dbc_id: + catalog_id: description: - The identifier of the parent Database Catalog attached to the Virtual Warehouse. - Required if C(state=present) type: str + aliases: + - dbc_id type: description: - The type of Virtual Warehouse to be created. @@ -239,7 +242,7 @@ # Delete a Virtual Warehouse - cloudera.cloud.dw_virtual_warehouse: cluster_id: example-cluster-id - id: example-virtual-warehouse-id + warehouse_id: example-virtual-warehouse-id state: absent ''' @@ -310,14 +313,14 @@ ''' -class DwVw(CdpModule): +class DwVirtualWarehouse(CdpModule): def __init__(self, module): - super(DwVw, self).__init__(module) + super(DwVirtualWarehouse, self).__init__(module) # Set variables - self.id = self._get_param('id') + self.warehouse_id = self._get_param('warehouse_id') self.cluster_id = self._get_param('cluster_id') - self.dbc_id = self._get_param('dbc_id') + self.dbc_id = self._get_param('catalog_id') self.type = self._get_param('type') self.name = self._get_param('name') self.template = self._get_param('template') @@ -345,13 +348,13 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.id is None: + if self.warehouse_id is None: vws = self.cdpy.dw.list_vws(cluster_id=self.cluster_id) for vw in vws: if self.name is not None and vw['name'] == self.name: self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw['id']) else: - self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) + self.target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.warehouse_id) if self.target is not None: # Begin Virtual Warehouse Exists @@ -422,9 +425,9 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(type='str', aliases=['vw_id']), + warehouse_id=dict(type='str', aliases=['vw_id', 'id']), cluster_id=dict(required=True, type='str'), - dbc_id=dict(type='str'), + catalog_id=dict(type='str', aliases=['dbc_id']), type = dict(type='str'), name = dict(type='str'), template=dict(type='str', choices=['xsmall', 'small', 'medium', 'large']), @@ -451,13 +454,13 @@ def main(): timeout = dict(type='int', aliases=['polling_timeout'], default=3600) ), required_if=[ - ['state', 'absent', ['id']], - ['state', 'present', ['dbc_id', 'type', 'name']] + ['state', 'absent', ['warehouse_id']], + ['state', 'present', ['catalog_id', 'type', 'name']] ], supports_check_mode=True ) - result = DwVw(module) + result = DwVirtualWarehouse(module) output = dict(changed=result.changed, virtual_warehouse=result.virtual_warehouse) if result.debug: diff --git a/plugins/modules/dw_virtual_warehouse_info.py b/plugins/modules/dw_virtual_warehouse_info.py index 6351f3b5..0f9808bc 100644 --- a/plugins/modules/dw_virtual_warehouse_info.py +++ b/plugins/modules/dw_virtual_warehouse_info.py @@ -35,29 +35,32 @@ requirements: - cdpy options: - id: + warehouse_id: description: - The identifier of the Virtual Warehouse. - Requires I(cluster_id). - - Mutually exclusive with I(name) and I(dbc_id). + - Mutually exclusive with I(name) and I(catalog_id). type: str aliases: - vw_id + - id cluster_id: description: - The identifier of the parent Data Warehouse Cluster of the Virtual Warehouse(s). type: str - dbc_id: + catalog_id: description: - The identifier of the parent Database Catalog attached to the Virtual Warehouse(s). - Requires I(cluster_id). - - Mutally exclusive with I(id) and I(name). + - Mutally exclusive with I(warehouse_id) and I(name). type: str + aliases: + - dbc_id name: description: - The name of the Virtual Warehouse. - Requires I(cluster_id). - - Mutually exclusive with I(id) and I(dbc_id). + - Mutually exclusive with I(warehouse_id) and I(catalog_id). type: str delay: description: @@ -90,12 +93,12 @@ # List all Virtual Warehouses associated with a Data Catalog - cloudera.cloud.dw_virtual_warehouse_info: cluster_id: example-cluster-id - dbc_id: example-data-catalog-id + catalog_id: example-data-catalog-id # Describe a Virtual Warehouse by ID - cloudera.cloud.dw_virtual_warehouse_info: cluster_id: example-cluster-id - id: example-virtual-warehouse-id + warehouse_id: example-virtual-warehouse-id # Describe a Virtual Warehouse by name - cloudera.cloud.dw_virtual_warehouse_info: @@ -171,14 +174,14 @@ ''' -class DwVwInfo(CdpModule): +class DwVirtualWarehouseInfo(CdpModule): def __init__(self, module): - super(DwVwInfo, self).__init__(module) + super(DwVirtualWarehouseInfo, self).__init__(module) # Set variables - self.id = self._get_param('id') + self.warehouse_id = self._get_param('warehouse_id') self.cluster_id = self._get_param('cluster_id') - self.dbc_id = self._get_param('dbc_id') + self.catalog_id = self._get_param('catalog_id') self.type = self._get_param('type') self.name = self._get_param('name') self.delay = self._get_param('delay') @@ -192,8 +195,8 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.id is not None: - target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.id) + if self.warehouse_id is not None: + target = self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=self.warehouse_id) if target is not None: self.virtual_warehouses.append(target) else: @@ -204,8 +207,8 @@ def process(self): self.virtual_warehouses.append( self.cdpy.dw.describe_vw(cluster_id=self.cluster_id, vw_id=vw['id']) ) - elif self.dbc_id is not None: - self.virtual_warehouses =[v for v in vws if v['dbcId'] == self.dbc_id] + elif self.catalog_id is not None: + self.virtual_warehouses =[v for v in vws if v['dbcId'] == self.catalog_id] else: self.virtual_warehouses = vws @@ -213,20 +216,20 @@ def process(self): def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - id=dict(type='str', aliases=['vw_id']), + warehouse_id=dict(type='str', aliases=['vw_id', 'id']), cluster_id=dict(required=True, type='str'), - dbc_id=dict(type='str'), + catalog_id=dict(type='str', aliases=['dbc_id']), name=dict(type='str'), delay=dict(type='int', aliases=['polling_delay'], default=15), timeout=dict(type='int', aliases=['polling_timeout'], default=3600) ), mutually_exclusive=[ - ['id', 'name', 'dbc_id'] + ['warehouse_id', 'name', 'catalog_id'] ], supports_check_mode=True ) - result = DwVwInfo(module) + result = DwVirtualWarehouseInfo(module) output = dict(changed=False, virtual_warehouses=result.virtual_warehouses) if result.debug: