From 2cb0389765514ec4d1c7d42b147626b4480304ea Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Fri, 29 Oct 2021 18:32:13 +0100 Subject: [PATCH 1/7] WIP Signed-off-by: Daniel Chaffelson --- plugins/modules/df_deployment.py | 404 ++++++++++++++++++ plugins/modules/df_deployment_info.py | 249 +++++++++++ plugins/modules/df_service.py | 2 +- .../{df_info.py => df_service_info.py} | 6 +- 4 files changed, 657 insertions(+), 4 deletions(-) create mode 100644 plugins/modules/df_deployment.py create mode 100644 plugins/modules/df_deployment_info.py rename plugins/modules/{df_info.py => df_service_info.py} (98%) diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py new file mode 100644 index 00000000..aa98075e --- /dev/null +++ b/plugins/modules/df_deployment.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} +#TODO: Update docs +DOCUMENTATION = r''' +--- +module: df_deployment +short_description: Enable or Disable CDP DataFlow Services +description: + - Enable or Disable CDP DataFlow Services +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + crn: + description: The name or crn of the CDP Environment to host the Dataflow Service + type: str + required: True + aliases: + - name + - env_crn + state: + description: + - The declarative state of the Dataflow Service + type: str + required: False + default: present + choices: + - present + - enabled + - absent + - disabled + nodes_min: + description: The minimum number of kubernetes nodes needed for the environment. + Note that the lowest minimum is 3 nodes. + type: int + default: 3 + required: False + aliases: + - min_k8s_node_count + nodes_max: + description: The maximum number of kubernetes nodes that environment may scale up under high-demand situations. + type: int + default: 3 + required: False + aliases: + - max_k8s_node_count + public_loadbalancer: + description: Indicates whether or not to use a public load balancer when deploying dependencies stack. + type: bool + required: False + aliases: + - use_public_load_balancer + ip_ranges: + description: The IP ranges authorized to connect to the Kubernetes API server + type: list + required: False + aliases: + - authorized_ip_ranges + persist: + description: Whether or not to retain the database records of related entities during removal. + type: bool + required: False + default: False + terminate: + description: Whether or not to terminate all deployments associated with this DataFlow deployment + type: bool + required: False + default: False + wait: + description: + - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state. + - If set to FALSE, the module will return immediately. + type: bool + required: False + default: True + delay: + description: + - The internal polling interval (in seconds) while the module waits for the Dataflow Service to achieve the + declared state. + type: int + required: False + default: 15 + aliases: + - polling_delay + timeout: + description: + - The internal polling timeout (in seconds) while the module waits for the Dataflow Service to achieve the + declared state. + type: int + required: False + default: 3600 + aliases: + - polling_timeout +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Create a Dataflow Service +- cloudera.cloud.df_deployment: + name: my-deployment + nodes_min: 3 + nodes_max: 10 + public_loadbalancer: True + ip_ranges: ['192.168.0.1/24'] + state: present + wait: yes + +# Remove a Dataflow Service with Async wait +- cloudera.cloud.df_deployment: + name: my-deployment + persist: False + state: absent + wait: yes + async: 3600 + poll: 0 + register: __my_teardown_request + +''' + +RETURN = r''' +--- +deployments: + description: The information about the named DataFlow Deployment or DataFlow Deployments + type: list + returned: always + elements: complex + contains: + crn: + description: The DataFlow Service's parent environment CRN. + returned: always + type: str + name: + description: The DataFlow Service's parent environment name. + returned: always + type: str + cloudPlatform: + description: The cloud platform of the environment. + returned: always + type: str + region: + description: The region of the environment. + returned: always + type: str + deploymentCount: + description: The deployment count. + returned: always + type: str + minK8sNodeCount: + description: The minimum number of Kubernetes nodes that need to be provisioned in the environment. + returned: always + type: int + maxK8sNodeCount: + description: The maximum number of kubernetes nodes that environment may scale up under high-demand situations. + returned: always + type: str + status: + description: The status of a DataFlow enabled environment. + returned: always + type: dict + contains: + state: + description: The state of the environment. + returned: always + type: str + message: + description: A status message for the environment. + returned: always + type: str + k8sNodeCount: + description: The number of kubernetes nodes currently in use by DataFlow for this environment. + returned: always + type: int + instanceType: + description: The instance type of the kubernetes nodes currently in use by DataFlow for this environment. + returned: always + type: str + dfLocalUrl: + description: The URL of the environment local DataFlow application. + returned: always + type: str + authorizedIpRanges: + description: The authorized IP Ranges. + returned: always + type: list + activeWarningAlertCount: + description: Current count of active alerts classified as a warning. + returned: always + type: int + activeErrorAlertCount: + description: Current count of active alerts classified as an error. + returned: always + type: int + clusterId: + description: Cluster id of the environment. + returned: if enabled + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFDeployment(CdpModule): + def __init__(self, module): + super(DFDeployment, self).__init__(module) + + # Set variables + self.env_crn = self._get_param('env_crn') + self.df_crn = self._get_param('df_crn') + self.nodes_min = self._get_param('nodes_min') + self.nodes_max = self._get_param('nodes_max') + self.public_loadbalancer = self._get_param('public_loadbalancer') + self.lb_ip_ranges = self._get_param('loadbalancer_ip_ranges') + self.kube_ip_ranges = self._get_param('kube_ip_ranges') + self.cluster_subnets = self._get_param('cluster_subnets') + self.lb_subnets = self._get_param('lb_subnets') + self.persist = self._get_param('persist') + self.terminate = self._get_param('terminate') + self.force = self._get_param('force') + + self.state = self._get_param('state') + self.wait = self._get_param('wait') + self.delay = self._get_param('delay') + self.timeout = self._get_param('timeout') + + # Initialize return values + self.deployment = {} + + # Initialize internal values + self.target = None + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + if self.env_crn is not None: + self.env_crn = self.cdpy.environments.resolve_environment_crn(self.env_crn) + if self.env_crn is not None or self.df_crn is not None: + self.target = self.cdpy.df.describe_deployment(env_crn=self.env_crn, df_crn=self.df_crn) + + if self.target is not None: + # DF Database Entry exists + if self.state in ['absent']: + if self.module.check_mode: + self.deployment = self.target + else: + self._disable_df() + elif self.state in ['present']: + self.module.warn( + "Dataflow Service already enabled and configuration validation and reconciliation is not supported;" + + "to change a Dataflow Service, explicitly disable and recreate the Service or use the UI") + if self.wait: + self.deployment = self._wait_for_enabled() + else: + self.module.fail_json( + msg="State %s is not valid for this module" % self.state) + else: + # Environment does not have DF database entry, and probably doesn't exist + if self.state in ['absent']: + self.module.log( + "Dataflow Service already disabled in CDP Environment %s" % self.env_crn) + elif self.state in ['present']: + if self.env_crn is None: + self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) + else: + # create DF Service + if not self.module.check_mode: + self.deployment = self.cdpy.df.enable_deployment( + env_crn=self.env_crn, + min_nodes=self.nodes_min, + max_nodes=self.nodes_max, + enable_public_ip=self.public_loadbalancer, + lb_ips=self.lb_ip_ranges, + kube_ips=self.kube_ip_ranges, + cluster_subnets=self.cluster_subnets, + lb_subnets=self.lb_subnets + ) + if self.wait: + self.deployment = self._wait_for_enabled() + else: + self.module.fail_json( + msg="State %s is not valid for this module" % self.state) + + def _wait_for_enabled(self): + return self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.df.describe_deployment, params=dict(env_crn=self.env_crn), + field=['status', 'state'], state=self.cdpy.sdk.STARTED_STATES, + delay=self.delay, timeout=self.timeout + ) + + def _disable_df(self): + # Attempt clean Disable, which also ensures we have tried at least once before we do a forced removal + if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES: + self.deployment = self.cdpy.df.disable_deployment( + df_crn=self.df_crn, + persist=self.persist, + terminate=self.terminate + ) + else: + self.module.warn("Attempting to disable DataFlow Service but state %s not in Removable States %s" + % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES)) + if self.wait: + # Wait for Clean Disable, if possible + self.deployment = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.df.describe_deployment, params=dict(df_crn=self.df_crn), + field=['status', 'state'], + state=self.cdpy.sdk.STOPPED_STATES + self.cdpy.sdk.REMOVABLE_STATES + [None], + delay=self.delay, timeout=self.timeout, ignore_failures=True + ) + else: + self.deployment = self.cdpy.df.describe_deployment(df_crn=self.df_crn) + # Check disable result against need for further forced delete action, in case it didn't work first time around + if self.deployment is not None: + if self.deployment['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES: + if self.force: + self.deployment = self.cdpy.df.reset_deployment( + df_crn=self.df_crn + ) + else: + self.module.fail_json(msg="DF Service Disable failed and Force delete not requested") + if self.wait: + self.deployment = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.df.describe_deployment, params=dict(df_crn=self.df_crn), + field=None, # This time we require removal or declare failure + delay=self.delay, timeout=self.timeout + ) + else: + self.deployment = self.cdpy.df.describe_deployment(df_crn=self.df_crn) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + env_crn=dict(type='str'), + df_crn=dict(type='str'), + nodes_min=dict(type='int', default=3, aliases=['min_k8s_node_count']), + nodes_max=dict(type='int', default=3, aliases=['max_k8s_node_count']), + public_loadbalancer=dict(type='bool', default=False, aliases=['use_public_load_balancer']), + loadbalancer_ip_ranges=dict(type='list', elements='str', default=None), + kube_ip_ranges=dict(type='list', elements='str', default=None), + cluster_subnets=dict(type='list', elements='str', default=None), + loadbalancer_subnets=dict(type='list', elements='str', default=None), + persist=dict(type='bool', default=False), + terminate=dict(type='bool', default=False), + state=dict(type='str', choices=['present', 'absent'], + default='present'), + force=dict(type='bool', default=False, aliases=['force_delete']), + wait=dict(type='bool', default=True), + delay=dict(type='int', aliases=['polling_delay'], default=15), + timeout=dict(type='int', aliases=['polling_timeout'], default=3600) + ), + supports_check_mode=True, + required_if=[ + ('state', 'present', ('env_crn', ), False), + ('state', 'absent', ('df_crn', ), False) + ] + ) + + result = DFService(module) + output = dict(changed=False, deployment=result.deployment) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_deployment_info.py b/plugins/modules/df_deployment_info.py new file mode 100644 index 00000000..684f5cf6 --- /dev/null +++ b/plugins/modules/df_deployment_info.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_deployment_info +short_description: Gather information about CDP DataFlow Deployments +description: + - Gather information about CDP DataFlow Deployments +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow Deployment will be described + - Must be the string CRN of the deployment + type: str + aliases: + - dep_crn + required: False + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List basic information about all DataFlow Deployments +- cloudera.cloud.df_deployment_info: + +# Gather detailed information about a named DataFlow Deployment using a name +- cloudera.cloud.df_deployment_info: + name: crn:cdp:df:region:tenant-uuid4:deployment:deployment-uuid4/deployment-uuid4 +''' + +RETURN = r''' +--- +deployments: + description: The information about the named DataFlow Deployment or DataFlow Deployments + type: list + returned: always + elements: complex + contains: + crn: + description: The DataFlow Deployment's CRN. + returned: always + type: str + name: + description: The DataFlow Deployment's name. + returned: always + type: str + status: + description: The status of a DataFlow enabled environment. + returned: always + type: dict + contains: + state: + description: The state of the Deployment. + returned: always + type: str + detailedState: + description: The state of the Deployment. + returned: always + type: str + message: + description: A status message for the Deployment. + returned: always + type: str + service: + description: Metadata about the parent DataFlow service. + returned: always + type: dict + contains: + crn: + description: The crn of the parent service. + returned: always + type: str + name: + description: The name of the parent environment. + returned: always + type: str + cloudProvider: + description: the cloud provider for the parent environment. + returned: always + type: str + region: + description: the region within the parent environment cloud provider. + returned: always + type: str + environmentCrn: + description: The CDP parent Environment CRN. + returned: always + type: str + updated: + description: Timestamp of the last time the deployment was modified. + returned: always + type: str + clusterSize: + description: The initial size of the deployment. + returned: always + type: str + flowVersionCrn: + description: The deployment's current flow version CRN. + returned: always + type: str + flowCrn: + description: The deployment's current flow CRN. + returned: always + type: str + nifiUrl: + description: The url to open the deployed flow in NiFi. + returned: always + type: str + autoscaleMaxNodes: + description: The maximum number of nodes that the deployment can scale up to, or null if autoscaling is not enabled for this deployment. + returned: always + type: complex + flowName: + description: The name of the flow. + returned: always + type: str + flowVersion: + description: The version of the flow. + returned: always + type: int + currentNodeCount: + description: The current node count. + returned: always + type: int + deployedByCrn: + description: The actor CRN of the person who deployed the flow. + returned: always + type: str + deployedByName: + description: The name of the person who deployed the flow. + returned: always + type: complex + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool + autoscaleMinNodes: + description: The minimum number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is true. + returned: always + type: int + activeWarningAlertCount: + description: Current count of active alerts classified as a warning. + returned: always + type: int + activeErrorAlertCount: + description: Current count of active alerts classified as an error. + returned: always + type: int + staticNodeCount: + description: The static number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is false. + returned: always + type: int + dfxLocalUrl: + description: Base URL to the DFX Local instance running this deployment. + returned: always + type: string + lastUpdatedByName: + description: The name of the person who last updated the deployment. + returned: always + type: string + configurationVersion: + description: The version of the configuration for this deployment. + returned: always + type: int +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFDeploymentInfo(CdpModule): + def __init__(self, module): + super(DFDeploymentInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + + # Initialize return values + self.deployments = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + if self.name is not None: + self.deployments = [self.cdpy.df.describe_deployment(dep_crn=self.name)] + else: + self.deployments = self.cdpy.df.list_deployments() + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str', aliases=['crn', 'dep_crn']) + ), + supports_check_mode=True, + mutually_exclusive=['name', 'df_crn', 'env_crn'] + ) + + result = DFDeploymentInfo(module) + output = dict(changed=False, deployments=result.deployments) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 772b1aec..6a34a1cd 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -147,7 +147,7 @@ RETURN = r''' --- -environments: +services: description: The information about the named DataFlow Service or DataFlow Services type: list returned: always diff --git a/plugins/modules/df_info.py b/plugins/modules/df_service_info.py similarity index 98% rename from plugins/modules/df_info.py rename to plugins/modules/df_service_info.py index e1a2debc..1479e3ce 100644 --- a/plugins/modules/df_info.py +++ b/plugins/modules/df_service_info.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module: df_info +module: df_service_info short_description: Gather information about CDP DataFlow Services description: - Gather information about CDP DataFlow Services @@ -169,9 +169,9 @@ ''' -class DFInfo(CdpModule): +class DFServiceInfo(CdpModule): def __init__(self, module): - super(DFInfo, self).__init__(module) + super(DFServiceInfo, self).__init__(module) # Set variables self.name = self._get_param('name') From 972623d48a5390200ad0f0c9beb44fdffdbae37d Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 16 Dec 2021 20:27:20 +0000 Subject: [PATCH 2/7] WIP Signed-off-by: Daniel Chaffelson --- plugins/modules/df_customflow_info.py | 197 +++++++++++++++ plugins/modules/df_deployment.py | 343 +++++++++++++++----------- plugins/modules/df_service.py | 17 +- plugins/modules/df_service_info.py | 10 +- 4 files changed, 422 insertions(+), 145 deletions(-) create mode 100644 plugins/modules/df_customflow_info.py diff --git a/plugins/modules/df_customflow_info.py b/plugins/modules/df_customflow_info.py new file mode 100644 index 00000000..d1bbc83b --- /dev/null +++ b/plugins/modules/df_customflow_info.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_customflow_info +short_description: Gather information about CDP DataFlow CustomFlow Definitions +description: + - Gather information about CDP DataFlow CustomFlow Definitions +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow Flow Definition will be described + type: str + required: False + include_details: + description: + - If set to false, only a summary of each flow is returned + type: bool + required: False + default: True + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List summary information about all Custom DataFlow Flow Definitions +- cloudera.cloud.df_customflow_info: + +# Gather summary information about a specific DataFlow Flow Definition using a name +- cloudera.cloud.df_customflow_info: + name: my-flow-name + include_details: False +''' + +RETURN = r''' +--- +catalog: + description: The listing of CustomFlow Definitions in the DataFlow Catalog in this CDP Tenant + type: list + returned: always + elements: complex + contains: + crn: + description: The DataFlow Flow Definition's CRN. + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + modifiedTimestamp: + description: THe timestamp the entry was last modified. + returned: always + type: int + versionCount: + description: The number of versions uploaded to the catalog. + returned: always + type: str + createdTimestamp: + description: The created timestamp. + returned: always + type: int + author: + description: Author of the most recent version. + returned: always + type: str + description: + description: The artifact description. + returned: always + type: str + versions: + description: The list of artifactDetail versions. + returned: always + type: array + contains: + crn: + description: The flow version CRN. + returned: always + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: always + type: str + author: + description: The author of the flow. + returned: always + type: str + version: + description: The version of the flow. + returned: always + type: int + timestamp: + description: The timestamp of the flow. + returned: always + type: int + deploymentCount: + description: The number of deployments of the flow. + returned: always + type: int + comments: + description: Comments about the flow. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFCustomFlowInfo(CdpModule): + def __init__(self, module): + super(DFCustomFlowInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.include_details = self._get_param('include_details') + + # Initialize internal values + self.listing = [] + + # Initialize return values + self.flows = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_flow_definitions(name=self.name) + if self.include_details: + self.flows = [ + self.cdpy.df.describe_customflow(x['crn']) + for x in self.listing + if x['artifactType'] == 'flow' # ReadyFlow have different fields + ] + else: + self.flows = self.listing + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str'), + include_details=dict(required=False, type='bool', default=True) + ), + supports_check_mode=True + ) + + result = DFCustomFlowInfo(module) + output = dict(changed=False, flows=result.flows) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py index aa98075e..5b048c24 100644 --- a/plugins/modules/df_deployment.py +++ b/plugins/modules/df_deployment.py @@ -33,61 +33,95 @@ requirements: - cdpy options: - crn: - description: The name or crn of the CDP Environment to host the Dataflow Service + name: + description: + - The name of the Deployed Flow, or Flow to be Deployed type: str - required: True - aliases: - - name - - env_crn - state: + required: False + dep_crn: + description: + - The crn of the Deployed Flow to be terminated + - Required if Name is not supplied for termination + type: str + required: False + df_crn: description: - - The declarative state of the Dataflow Service + - The crn of the Dataflow Service + - Required if the df_name is not supplied type: str required: False - default: present - choices: - - present - - enabled - - absent - - disabled - nodes_min: - description: The minimum number of kubernetes nodes needed for the environment. - Note that the lowest minimum is 3 nodes. - type: int - default: 3 + df_name: + description: + - The Name of the Dataflow Service + - Required if the df_crn is not supplied + type: str required: False - aliases: - - min_k8s_node_count - nodes_max: - description: The maximum number of kubernetes nodes that environment may scale up under high-demand situations. + flow_ver_crn: + description: + - The crn of the specific Version of the Flow to be Deployed + - Required for creating a Deployment if flow_name is not supplied + type: str + required: False + flow_name: + description: + - The Name of the Flow to be Deployed + - Required for creating a Deployment if flow_ver_crn is not supplied + type: str + required: False + flow_ver_crn: + description: + - The crn of the specific Version of the Flow to be Deployed + - Required for creating a Deployment if flow_name is not supplied + type: str + required: False + flow_ver: + description: + - The Version number of the Flow to be Deployed + - If not supplied, the latest version available will be Deployed type: int - default: 3 required: False - aliases: - - max_k8s_node_count - public_loadbalancer: - description: Indicates whether or not to use a public load balancer when deploying dependencies stack. - type: bool + default: newest + size: + description: + - The Size of the Pod for the Flow to be Deployed into + type: str + default: SMALL + options: + - EXTRA_SMALL + - SMALL + - MEDIUM + - LARGE required: False - aliases: - - use_public_load_balancer - ip_ranges: - description: The IP ranges authorized to connect to the Kubernetes API server - type: list + static_node_count: + description: + - The number of nodes to build the Pod on if not using Autoscaling + type: int required: False - aliases: - - authorized_ip_ranges - persist: - description: Whether or not to retain the database records of related entities during removal. + default: 1 + autoscale: + description: + - Whether to use autoscaling of pods for this Deployment type: bool required: False default: False - terminate: - description: Whether or not to terminate all deployments associated with this DataFlow deployment - type: bool + autoscale_nodes_min: + description: + - The minimum number of nodes to use when Autoscaling + type: int required: False - default: False + default: 1 + autoscale_nodes_max: + description: + - The maximum number of nodes to use when Autoscaling + type: int + required: False + default: 3 + nifi_ver: + description: + - The specific version of NiFi to use in the Deployment + type: str + required: False + default: latest wait: description: - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state. @@ -95,6 +129,22 @@ type: bool required: False default: True + autostart_flow: + description: + - Whether to automatically start the Flow once Deployment is complete + type: bool + required: False + default: True + parameter_groups: + description: + - Definitions of Parameters to apply to the Deployed Flow + type: dict + required: False + kpis: + description: + - Definitions of KPIs to apply to the Deployed Flow + type: list + required: False delay: description: - The internal polling interval (in seconds) while the module waits for the Dataflow Service to achieve the @@ -123,20 +173,14 @@ EXAMPLES = r''' # Note: These examples do not set authentication details. -# Create a Dataflow Service +# Deploy a Dataflow with defaults - cloudera.cloud.df_deployment: - name: my-deployment - nodes_min: 3 - nodes_max: 10 - public_loadbalancer: True - ip_ranges: ['192.168.0.1/24'] - state: present - wait: yes + name: my-flow # Remove a Dataflow Service with Async wait - cloudera.cloud.df_deployment: - name: my-deployment - persist: False + name: my-flow-name + df_name: my-env-name state: absent wait: yes async: 3600 @@ -239,18 +283,22 @@ def __init__(self, module): super(DFDeployment, self).__init__(module) # Set variables - self.env_crn = self._get_param('env_crn') + self.name = self._get_param('name') + self.dep_crn = self._get_param('dep_crn') self.df_crn = self._get_param('df_crn') - self.nodes_min = self._get_param('nodes_min') - self.nodes_max = self._get_param('nodes_max') - self.public_loadbalancer = self._get_param('public_loadbalancer') - self.lb_ip_ranges = self._get_param('loadbalancer_ip_ranges') - self.kube_ip_ranges = self._get_param('kube_ip_ranges') - self.cluster_subnets = self._get_param('cluster_subnets') - self.lb_subnets = self._get_param('lb_subnets') - self.persist = self._get_param('persist') - self.terminate = self._get_param('terminate') - self.force = self._get_param('force') + self.df_name = self._get_param('df_name') + self.flow_ver_crn = self._get_param('flow_ver_crn') + self.flow_name = self._get_param('flow_name') + self.flow_ver = self._get_param('flow_ver') + self.size = self._get_param('size') + self.static_node_count = self._get_param('static_node_count') + self.autoscale_enabled = self._get_param('autoscale_enabled') + self.autoscale_nodes_min = self._get_param('autoscale_nodes_min') + self.autoscale_nodes_max = self._get_param('autoscale_nodes_max') + self.nifi_ver = self._get_param('nifi_ver') + self.autostart_flow = self._get_param('autostart_flow') + self.parameter_groups = self._get_param('parameter_groups') + self.kpis = self._get_param('kpis') self.state = self._get_param('state') self.wait = self._get_param('wait') @@ -259,140 +307,153 @@ def __init__(self, module): # Initialize return values self.deployment = {} + self.changed = False # Initialize internal values - self.target = None + self.target = {} # Execute logic process self.process() @CdpModule._Decorators.process_debug def process(self): - if self.env_crn is not None: - self.env_crn = self.cdpy.environments.resolve_environment_crn(self.env_crn) - if self.env_crn is not None or self.df_crn is not None: - self.target = self.cdpy.df.describe_deployment(env_crn=self.env_crn, df_crn=self.df_crn) - + # Prepare information + if self.df_crn is None: + self.df_crn = self.cdpy.df.resolve_service_crn_from_name(self.df_name) + if self.df_crn is None: + self.module.fail_json( + msg="Either df_crn must be supplied or resolvable from df_name") + if self.dep_crn is not None: + self.target = self.cdpy.df.describe_deployment(dep_crn=self.dep_crn) + elif self.name is not None and self.df_crn is not None: + self.target = self.cdpy.df.describe_deployment(df_crn=self.df_crn, name=self.name) + if self.target is not None: + self.dep_crn = self.target['crn'] + # Process execution if self.target is not None: - # DF Database Entry exists + # DF Deployment exists if self.state in ['absent']: + # Existing Deployment to be removed if self.module.check_mode: self.deployment = self.target else: - self._disable_df() + self._terminate_deployment() + # Existing deployment to be retained elif self.state in ['present']: self.module.warn( - "Dataflow Service already enabled and configuration validation and reconciliation is not supported;" + - "to change a Dataflow Service, explicitly disable and recreate the Service or use the UI") + "Dataflow Deployment already exists and configuration validation and reconciliation " + + "is not supported;" + + "to change a Deployment, explicitly terminate and recreate it or use the UI") if self.wait: - self.deployment = self._wait_for_enabled() + self.deployment = self._wait_for_deployed() else: self.module.fail_json( msg="State %s is not valid for this module" % self.state) else: - # Environment does not have DF database entry, and probably doesn't exist + # Deployment CRN not found in Tenant, and probably doesn't exist if self.state in ['absent']: self.module.log( - "Dataflow Service already disabled in CDP Environment %s" % self.env_crn) + "Dataflow Deployment not found in CDP Tenant %s" % self.dep_crn) + # Deployment to be created elif self.state in ['present']: - if self.env_crn is None: - self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) + # create Deployment + if not self.module.check_mode: + self._create_deployment() + if self.wait: + self.deployment = self._wait_for_deployed() else: - # create DF Service - if not self.module.check_mode: - self.deployment = self.cdpy.df.enable_deployment( - env_crn=self.env_crn, - min_nodes=self.nodes_min, - max_nodes=self.nodes_max, - enable_public_ip=self.public_loadbalancer, - lb_ips=self.lb_ip_ranges, - kube_ips=self.kube_ip_ranges, - cluster_subnets=self.cluster_subnets, - lb_subnets=self.lb_subnets - ) - if self.wait: - self.deployment = self._wait_for_enabled() + pass # Check mode can return the described deployment else: self.module.fail_json( msg="State %s is not valid for this module" % self.state) - def _wait_for_enabled(self): + def _create_deployment(self): + if self.flow_ver_crn is None: + # flow_name must be populated if flow_ver_crn is None + self.flow_ver_crn = self.cdpy.df.get_version_crn_from_flow_definition(self.flow_name, self.flow_ver) + self.deployment = self.cdpy.df.create_deployment( + df_crn=self.df_crn, + flow_ver_crn=self.flow_ver_crn, + deployment_name=self.name, + size_name=self.size, + static_node_count=self.static_node_count, + autoscale_enabled=self.autoscale_enabled, + autoscale_nodes_min=self.autoscale_nodes_min, + autoscale_nodes_max=self.autoscale_nodes_max, + nifi_ver=self.nifi_ver, + autostart_flow=self.autostart_flow, + parameter_groups=self.parameter_groups, + kpis=self.kpis, + ) + self.changed = True + + def _wait_for_deployed(self): return self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.df.describe_deployment, params=dict(env_crn=self.env_crn), + describe_func=self.cdpy.df.describe_deployment, + params=dict(dep_crn=self.dep_crn, df_crn=self.df_crn, name=self.name), field=['status', 'state'], state=self.cdpy.sdk.STARTED_STATES, delay=self.delay, timeout=self.timeout ) - def _disable_df(self): - # Attempt clean Disable, which also ensures we have tried at least once before we do a forced removal + def _terminate_deployment(self): if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES: - self.deployment = self.cdpy.df.disable_deployment( - df_crn=self.df_crn, - persist=self.persist, - terminate=self.terminate + self.deployment = self.cdpy.df.terminate_deployment( + dep_crn=self.dep_crn ) + self.changed = True else: - self.module.warn("Attempting to disable DataFlow Service but state %s not in Removable States %s" + self.module.warn("Attempting to disable DataFlow Deployment but state %s not in Removable States %s" % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES)) if self.wait: - # Wait for Clean Disable, if possible self.deployment = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.df.describe_deployment, params=dict(df_crn=self.df_crn), - field=['status', 'state'], - state=self.cdpy.sdk.STOPPED_STATES + self.cdpy.sdk.REMOVABLE_STATES + [None], - delay=self.delay, timeout=self.timeout, ignore_failures=True + describe_func=self.cdpy.df.describe_deployment, + params=dict(dep_crn=self.dep_crn), field=None, + delay=self.delay, timeout=self.timeout ) else: - self.deployment = self.cdpy.df.describe_deployment(df_crn=self.df_crn) - # Check disable result against need for further forced delete action, in case it didn't work first time around - if self.deployment is not None: - if self.deployment['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES: - if self.force: - self.deployment = self.cdpy.df.reset_deployment( - df_crn=self.df_crn - ) - else: - self.module.fail_json(msg="DF Service Disable failed and Force delete not requested") - if self.wait: - self.deployment = self.cdpy.sdk.wait_for_state( - describe_func=self.cdpy.df.describe_deployment, params=dict(df_crn=self.df_crn), - field=None, # This time we require removal or declare failure - delay=self.delay, timeout=self.timeout - ) - else: - self.deployment = self.cdpy.df.describe_deployment(df_crn=self.df_crn) + self.deployment = self.cdpy.df.describe_deployment(dep_crn=self.dep_crn) def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( - env_crn=dict(type='str'), - df_crn=dict(type='str'), - nodes_min=dict(type='int', default=3, aliases=['min_k8s_node_count']), - nodes_max=dict(type='int', default=3, aliases=['max_k8s_node_count']), - public_loadbalancer=dict(type='bool', default=False, aliases=['use_public_load_balancer']), - loadbalancer_ip_ranges=dict(type='list', elements='str', default=None), - kube_ip_ranges=dict(type='list', elements='str', default=None), - cluster_subnets=dict(type='list', elements='str', default=None), - loadbalancer_subnets=dict(type='list', elements='str', default=None), - persist=dict(type='bool', default=False), - terminate=dict(type='bool', default=False), + name=dict(type='str'), + df_crn=dict(type='str', default=None), + df_name=dict(type='str', default=None), + dep_crn=dict(type='str', default=None), + flow_ver_crn=dict(type='str', default=None), + flow_name=dict(type='str', default=None), + flow_ver=dict(type='int', default=None), + size=dict(type='str', + choices=['EXTRA_SMALL', 'SMALL', 'MEDIUM', 'LARGE'], + default='EXTRA_SMALL', aliases=['size_name']), + static_node_count=dict(type='int', default=1), + autoscale=dict(type='bool', default=False, aliases=['autoscale_enabled']), + autoscale_nodes_min=dict(type='int', default=1), + autoscale_nodes_max=dict(type='int', default=3), + nifi_ver=dict(type='str', default=None), + autostart_flow=dict(type='bool', default=True), + parameter_groups=dict(type='dict', default=None), + kpis=dict(type='list', default=None), state=dict(type='str', choices=['present', 'absent'], default='present'), - force=dict(type='bool', default=False, aliases=['force_delete']), wait=dict(type='bool', default=True), delay=dict(type='int', aliases=['polling_delay'], default=15), timeout=dict(type='int', aliases=['polling_timeout'], default=3600) ), supports_check_mode=True, + required_one_of=[ + ['df_crn', 'df_name'] + ], required_if=[ - ('state', 'present', ('env_crn', ), False), - ('state', 'absent', ('df_crn', ), False) - ] + ['state', 'absent', ['dep_crn', 'name'], True], # One of for termination + ['state', 'present', ['flow_ver_crn', 'flow_name'], True], # One of + ['state', 'present', ['name']] + ], ) - result = DFService(module) - output = dict(changed=False, deployment=result.deployment) + result = DFDeployment(module) + output = dict(changed=result.changed, deployment=result.deployment) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 6a34a1cd..960a161c 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -88,6 +88,10 @@ type: bool required: False default: False + tags: + description: Tags to apply to the DataFlow Service + type: dict + required: False wait: description: - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state. @@ -251,6 +255,7 @@ def __init__(self, module): self.persist = self._get_param('persist') self.terminate = self._get_param('terminate') self.force = self._get_param('force') + self.tags = self._get_param('tags') self.state = self._get_param('state') self.wait = self._get_param('wait') @@ -259,6 +264,7 @@ def __init__(self, module): # Initialize return values self.service = {} + self.changed = False # Initialize internal values self.target = None @@ -298,6 +304,7 @@ def process(self): if self.env_crn is None: self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) else: + df_tags = [{'key': x, 'value': self.tags[x]} for x in self.tags] if self.tags is not None else None # create DF Service if not self.module.check_mode: self.service = self.cdpy.df.enable_service( @@ -307,9 +314,11 @@ def process(self): enable_public_ip=self.public_loadbalancer, lb_ips=self.lb_ip_ranges, kube_ips=self.kube_ip_ranges, + tags=df_tags, cluster_subnets=self.cluster_subnets, lb_subnets=self.lb_subnets ) + self.changed = True if self.wait: self.service = self._wait_for_enabled() else: @@ -331,6 +340,10 @@ def _disable_df(self): persist=self.persist, terminate=self.terminate ) + self.changed = True + elif self.target['status']['state'] in self.cdpy.sdk.TERMINATION_STATES: + self.module.warn("DataFlow Service is already Disabling, skipping termination request") + pass else: self.module.warn("Attempting to disable DataFlow Service but state %s not in Removable States %s" % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES)) @@ -351,6 +364,7 @@ def _disable_df(self): self.service = self.cdpy.df.reset_service( df_crn=self.df_crn ) + self.changed = True else: self.module.fail_json(msg="DF Service Disable failed and Force delete not requested") if self.wait: @@ -377,6 +391,7 @@ def main(): loadbalancer_subnets=dict(type='list', elements='str', default=None), persist=dict(type='bool', default=False), terminate=dict(type='bool', default=False), + tags=dict(required=False, type='dict', default=None), state=dict(type='str', choices=['present', 'absent'], default='present'), force=dict(type='bool', default=False, aliases=['force_delete']), @@ -392,7 +407,7 @@ def main(): ) result = DFService(module) - output = dict(changed=False, service=result.service) + output = dict(changed=result.changed, service=result.service) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/df_service_info.py b/plugins/modules/df_service_info.py index 1479e3ce..787727e1 100644 --- a/plugins/modules/df_service_info.py +++ b/plugins/modules/df_service_info.py @@ -82,14 +82,18 @@ RETURN = r''' --- -environments: +services: description: The information about the named DataFlow Service or DataFlow Services type: list returned: always elements: complex contains: crn: - description: The DataFlow Service's parent environment CRN. + description: The DataFlow Service's CRN. + returned: always + type: str + environmentCrn: + description: The DataFlow Service's Parent Environment CRN. returned: always type: str name: @@ -209,7 +213,7 @@ def main(): mutually_exclusive=['name', 'df_crn', 'env_crn'] ) - result = DFInfo(module) + result = DFServiceInfo(module) output = dict(changed=False, services=result.services) if result.debug: From d1b17672cfce7f2b30c3e7468f4bb5e8360c296c Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Tue, 21 Dec 2021 18:11:08 +0000 Subject: [PATCH 3/7] Separate considerations for readyflows and customflows Add support for readyflows Remove comment support from df_service until string limitations are inline with rest of platform Add support for terminating deployed flows when disabling the DFX Service Signed-off-by: Daniel Chaffelson --- plugins/modules/df_customflow_info.py | 2 +- plugins/modules/df_deployment.py | 4 +- plugins/modules/df_readyflow.py | 272 ++++++++++++++++++++++++++ plugins/modules/df_readyflow_info.py | 244 +++++++++++++++++++++++ plugins/modules/df_service.py | 3 +- 5 files changed, 521 insertions(+), 4 deletions(-) create mode 100644 plugins/modules/df_readyflow.py create mode 100644 plugins/modules/df_readyflow_info.py diff --git a/plugins/modules/df_customflow_info.py b/plugins/modules/df_customflow_info.py index d1bbc83b..92306186 100644 --- a/plugins/modules/df_customflow_info.py +++ b/plugins/modules/df_customflow_info.py @@ -66,7 +66,7 @@ RETURN = r''' --- -catalog: +flows: description: The listing of CustomFlow Definitions in the DataFlow Catalog in this CDP Tenant type: list returned: always diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py index 5b048c24..dfed614a 100644 --- a/plugins/modules/df_deployment.py +++ b/plugins/modules/df_deployment.py @@ -306,7 +306,7 @@ def __init__(self, module): self.timeout = self._get_param('timeout') # Initialize return values - self.deployment = {} + self.deployment = None self.changed = False # Initialize internal values @@ -335,6 +335,8 @@ def process(self): if self.state in ['absent']: # Existing Deployment to be removed if self.module.check_mode: + self.module.log( + "Check mode enabled, skipping termination of Deployment %s" % self.dep_crn) self.deployment = self.target else: self._terminate_deployment() diff --git a/plugins/modules/df_readyflow.py b/plugins/modules/df_readyflow.py new file mode 100644 index 00000000..4d143c04 --- /dev/null +++ b/plugins/modules/df_readyflow.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_readyflow +short_description: Import or Delete ReadyFlows from your CDP Tenant +description: + - Import or Delete ReadyFlows from your CDP Tenant +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow ReadyFlow Definition will be described + type: str + required: False + state: + description: + - The declarative state of the ReadyFlow + type: str + required: False + default: present + choices: + - present + - absent + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Import a ReadyFlow into your CDP Tenant +- cloudera.cloud.df_readyflow: + name: my-readyflow-name + +# Delete an added ReadyFlow from your CDP Tenant +- cloudera.cloud.df_readyflow: + name: my-readyflow-name + state: absent +''' + +RETURN = r''' +--- +readyflow: + description: The listing of ReadyFlow Definitions in the DataFlow Catalog in this CDP Tenant + type: list + returned: always + elements: complex + contains: + readyflowCrn: + description: + - The DataFlow readyflow Definition's CRN. + - Use this readyflowCrn to address this object + returned: always + type: str + readyflow: + description: The details of the ReadyFlow object + type: dict + returned: always + elements: complex + contains: + readyflowCrn: + description: + - The general base crn of this ReadyFlow + - Different to the unique readyflowCrn containing a UUID4 + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + author: + description: Author of the most recent version. + returned: always + type: str + summary: + description: The ready flow summary (short). + returned: always + type: str + description: + description: The ready flow description (long). + returned: always + type: str + documentationLink: + description: A link to the ready flow documentation. + returned: always + type: str + notes: + description: Optional notes about the ready flow. + returned: always + type: str + source: + description: The ready flow data source. + returned: always + type: str + sourceDataFormat: + description: The ready flow data source format. + returned: always + type: str + destination: + description: The ready flow data destination. + returned: always + type: str + destinationDataFormat: + description: The ready flow data destination format. + returned: always + type: str + imported: + description: Whether the ready flow has been imported into the current account. + returned: always + type: bool + modifiedTimestamp: + description: THe timestamp the entry was last modified. + returned: always + type: int + versions: + description: The list of artifactDetail versions. + returned: When imported is True + type: array + contains: + crn: + description: The artifact version CRN. + returned: always + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: always + type: str + author: + description: The author of the artifact. + returned: always + type: str + version: + description: The version of the artifact. + returned: always + type: int + timestamp: + description: The timestamp of the artifact. + returned: always + type: int + deploymentCount: + description: The number of deployments of the artifact. + returned: always + type: int + comments: + description: Comments about the version. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFReadyFlow(CdpModule): + def __init__(self, module): + super(DFReadyFlow, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.include_details = self._get_param('include_details') + self.state = self._get_param('state') + + # Initialize internal values + self.target = None + self.listing = None + + # Initialize return values + self.readyflow = None + self.changed = False + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_readyflows(name=self.name) + if not self.listing: # return is list with one item if name exists, as name is unique + self.module.fail_json( + msg="ReadyFlow with Name %s is not found" % self.name) + else: + self.target = self.listing[0] + if self.target['imported']: # field is bool + if self.state == 'present': + # ReadyFlow is imported and should be left alone + # helpfully return the detailed description + self.readyflow = self.cdpy.df.describe_added_readyflow( + def_crn=self.target['importedArtifactCrn'] + ) + if self.state == 'absent': + if not self.module.check_mode: + # ReadyFlow is imported and should be deleted + self.readyflow = self.cdpy.df.delete_added_readyflow( + def_crn=self.target['importedArtifactCrn'] + ) + self.changed = True + else: + self.module.log( + "Check mode enabled, skipping deletion of %s" % self.name) + else: + if self.state == 'present': + # ReadyFlow should be imported + if not self.module.check_mode: + self.readyflow = self.cdpy.df.import_readyflow( + def_crn=self.target['readyflowCrn'] + ) + self.changed = True + else: + self.module.log( + "Check mode enabled, skipping import of %s" % self.name) + if self.state == 'absent': + # ReadyFlow is not imported and should stay that way + self.module.log( + "ReadyFlow already not imported to CDP Tenant %s" % self.name) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=True, type='str'), + state=dict(type='str', choices=['present', 'absent'], + default='present'), + ), + supports_check_mode=True + ) + + result = DFReadyFlow(module) + output = dict(changed=result.changed, readyflow=result.readyflow) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_readyflow_info.py b/plugins/modules/df_readyflow_info.py new file mode 100644 index 00000000..5aa414b0 --- /dev/null +++ b/plugins/modules/df_readyflow_info.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_readyflow_info +short_description: Gather information about CDP DataFlow CustomFlow Definitions +description: + - Gather information about CDP DataFlow ReadyFlow Definitions +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow ReadyFlow Definition will be described + type: str + required: False + include_details: + description: + - If set to false, only a summary of each ReadyFlow Definition is returned + type: bool + required: False + default: True + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List summary information about all Custom DataFlow ReadyFlow Definitions +- cloudera.cloud.df_readyflow_info: + +# Gather summary information about a specific DataFlow Flow Definition using a name +- cloudera.cloud.df_readyflow_info: + name: my-flow-name + include_details: False +''' + +RETURN = r''' +--- +flows: + description: The listing of ReadyFlow Definitions in the DataFlow Catalog in this CDP Tenant + type: list + returned: always + elements: complex + contains: + readyflowCrn: + description: + - The DataFlow readyflow Definition's CRN. + - Use this readyflowCrn to address this object + returned: always + type: str + readyflow: + description: The details of the ReadyFlow object + type: dict + returned: always + elements: complex + contains: + readyflowCrn: + description: + - The general base crn of this ReadyFlow + - Different to the unique readyflowCrn containing a UUID4 + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + author: + description: Author of the most recent version. + returned: always + type: str + summary: + description: The ready flow summary (short). + returned: always + type: str + description: + description: The ready flow description (long). + returned: always + type: str + documentationLink: + description: A link to the ready flow documentation. + returned: always + type: str + notes: + description: Optional notes about the ready flow. + returned: always + type: str + source: + description: The ready flow data source. + returned: always + type: str + sourceDataFormat: + description: The ready flow data source format. + returned: always + type: str + destination: + description: The ready flow data destination. + returned: always + type: str + destinationDataFormat: + description: The ready flow data destination format. + returned: always + type: str + imported: + description: Whether the ready flow has been imported into the current account. + returned: always + type: bool + modifiedTimestamp: + description: THe timestamp the entry was last modified. + returned: always + type: int + versions: + description: The list of artifactDetail versions. + returned: When imported is True + type: array + contains: + crn: + description: The artifact version CRN. + returned: always + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: always + type: str + author: + description: The author of the artifact. + returned: always + type: str + version: + description: The version of the artifact. + returned: always + type: int + timestamp: + description: The timestamp of the artifact. + returned: always + type: int + deploymentCount: + description: The number of deployments of the artifact. + returned: always + type: int + comments: + description: Comments about the version. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFReadyFlowInfo(CdpModule): + def __init__(self, module): + super(DFReadyFlowInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.include_details = self._get_param('include_details') + + # Initialize internal values + self.listing = [] + + # Initialize return values + self.flows = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_readyflows(name=self.name) + if self.include_details and self.listing: + self.flows = [] + for this_readyflow in self.listing: + if this_readyflow['imported']: + self.flows.append( + self.cdpy.df.describe_added_readyflow( + def_crn=this_readyflow['importedArtifactCrn'] + ) + ) + else: + self.flows.append( + self.cdpy.df.describe_readyflow( + def_crn=this_readyflow['readyflowCrn'] + ) + ) + else: + self.flows = self.listing + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str'), + include_details=dict(required=False, type='bool', default=True) + ), + supports_check_mode=True + ) + + result = DFReadyFlowInfo(module) + output = dict(changed=False, flows=result.flows) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 960a161c..64526653 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -304,7 +304,6 @@ def process(self): if self.env_crn is None: self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env) else: - df_tags = [{'key': x, 'value': self.tags[x]} for x in self.tags] if self.tags is not None else None # create DF Service if not self.module.check_mode: self.service = self.cdpy.df.enable_service( @@ -314,7 +313,7 @@ def process(self): enable_public_ip=self.public_loadbalancer, lb_ips=self.lb_ip_ranges, kube_ips=self.kube_ip_ranges, - tags=df_tags, + # tags=self.tags, # Currently overstrict blocking of values cluster_subnets=self.cluster_subnets, lb_subnets=self.lb_subnets ) From a735e0754a942de23c928bc0edddb2031aabf87f Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Mon, 10 Jan 2022 14:21:44 +0000 Subject: [PATCH 4/7] DFX parameter groups submission set to arrays Added DFX to cloudera.exe.info role Set DFX Deployments to only be attempted when defined Added Example DFX Flow deployment to examples/cdf definition in cloudera-deploy Signed-off-by: Daniel Chaffelson --- plugins/modules/df_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py index dfed614a..3d2940ee 100644 --- a/plugins/modules/df_deployment.py +++ b/plugins/modules/df_deployment.py @@ -138,7 +138,7 @@ parameter_groups: description: - Definitions of Parameters to apply to the Deployed Flow - type: dict + type: array required: False kpis: description: @@ -435,7 +435,7 @@ def main(): autoscale_nodes_max=dict(type='int', default=3), nifi_ver=dict(type='str', default=None), autostart_flow=dict(type='bool', default=True), - parameter_groups=dict(type='dict', default=None), + parameter_groups=dict(type='list', default=None), kpis=dict(type='list', default=None), state=dict(type='str', choices=['present', 'absent'], default='present'), From 27b15060859fa5481e73427faefbb961cc14693b Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Mon, 24 Jan 2022 10:30:33 +0000 Subject: [PATCH 5/7] Fix Documentation for DF Deployments Signed-off-by: Daniel Chaffelson --- plugins/modules/df_customflow_info.py | 4 +- plugins/modules/df_deployment.py | 148 ++++++++++++++++++-------- plugins/modules/df_deployment_info.py | 19 ++-- plugins/modules/df_readyflow.py | 17 ++- plugins/modules/df_readyflow_info.py | 8 +- plugins/modules/df_service.py | 18 ++-- 6 files changed, 142 insertions(+), 72 deletions(-) diff --git a/plugins/modules/df_customflow_info.py b/plugins/modules/df_customflow_info.py index 92306186..84e5d19d 100644 --- a/plugins/modules/df_customflow_info.py +++ b/plugins/modules/df_customflow_info.py @@ -46,7 +46,7 @@ default: True notes: - - This feature this module is for is in Technical Preview + - The feature this module is for is in Technical Preview extends_documentation_fragment: - cloudera.cloud.cdp_sdk_options - cloudera.cloud.cdp_auth_options @@ -81,7 +81,7 @@ returned: always type: str modifiedTimestamp: - description: THe timestamp the entry was last modified. + description: The timestamp the entry was last modified. returned: always type: int versionCount: diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py index 3d2940ee..94159b70 100644 --- a/plugins/modules/df_deployment.py +++ b/plugins/modules/df_deployment.py @@ -191,38 +191,17 @@ RETURN = r''' --- -deployments: - description: The information about the named DataFlow Deployment or DataFlow Deployments - type: list +deployment: + description: The information about the named DataFlow Deployment + type: dict returned: always elements: complex - contains: crn: - description: The DataFlow Service's parent environment CRN. + description: The deployment CRN. returned: always type: str name: - description: The DataFlow Service's parent environment name. - returned: always - type: str - cloudPlatform: - description: The cloud platform of the environment. - returned: always - type: str - region: - description: The region of the environment. - returned: always - type: str - deploymentCount: - description: The deployment count. - returned: always - type: str - minK8sNodeCount: - description: The minimum number of Kubernetes nodes that need to be provisioned in the environment. - returned: always - type: int - maxK8sNodeCount: - description: The maximum number of kubernetes nodes that environment may scale up under high-demand situations. + description: The deployment name. returned: always type: str status: @@ -230,30 +209,101 @@ returned: always type: dict contains: - state: - description: The state of the environment. + detailedState: + description: The detailed state that the deployment is currently in. returned: always type: str message: description: A status message for the environment. returned: always type: str - k8sNodeCount: - description: The number of kubernetes nodes currently in use by DataFlow for this environment. + state: + description: The state that the deployment is currently in + returned: always + type: str + service: + description: Metadata about the DataFlow service. + returned: always + type: dict + contains: + crn: + description: The crn of the DataFlow service. + returned: always + type: str + name: + description: The name of the CDP Environment. + returned: always + type: str + cloudProvider: + description: The cloud provider + returned: always + type: str + region: + description: The region within the cloud provider + returned: always + type: str + environmentCrn: + description: The CDP Environment CRN + returned: always + type: str + updated: + description: Timestamp of the last time the deployment was modified. + returned: always + type: int + clusterSize: + description: The initial size of the deployment. + returned: always + type: str + flowVersionCrn: + description: The deployment's current flow version CRN. + returned: always + type: str + flowCrn: + description: The deployment's current flow CRN. returned: always type: int - instanceType: - description: The instance type of the kubernetes nodes currently in use by DataFlow for this environment. + nifiUrl: + description: The url to open the deployed flow in NiFi. returned: always type: str - dfLocalUrl: - description: The URL of the environment local DataFlow application. + autoscaleMaxNodes: + description: The maximum number of nodes that the deployment can scale up to, or null if autoscaling is not enabled for this deployment. + returned: always + type: int + flowName: + description: The name of the flow. returned: always type: str - authorizedIpRanges: - description: The authorized IP Ranges. + flowVersion: + description: The version of the flow. + returned: always + type: int + currentNodeCount: + description: The current node count. returned: always - type: list + type: int + deployedByCrn: + description: The actor CRN of the person who deployed the flow. + returned: always + type: str + deployedByName: + description: The name of the person who deployed the flow. + returned: always + type: str + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool + autoscaleMinNodes: + description: + - The minimum number of nodes that the deployment will allocate. + - May only be specified when autoscalingEnabled is true. + returned: always + type: int + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool activeWarningAlertCount: description: Current count of active alerts classified as a warning. returned: always @@ -262,10 +312,24 @@ description: Current count of active alerts classified as an error. returned: always type: int - clusterId: - description: Cluster id of the environment. - returned: if enabled + staticNodeCount: + description: + - The static number of nodes that the deployment will allocate. + - May only be specified when autoscalingEnabled is false. + returned: always + type: int + dfxLocalUrl: + description: Base URL to the dfx-local instance running this deployment. + returned: always + type: str + lastUpdatedByName: + description: The name of the person who last updated the deployment. + returned: always type: str + configurationVersion: + description: The version of the configuration for this deployment. + returned: always + type: int sdk_out: description: Returns the captured CDP SDK log. returned: when supported @@ -340,8 +404,8 @@ def process(self): self.deployment = self.target else: self._terminate_deployment() - # Existing deployment to be retained elif self.state in ['present']: + # Existing deployment to be retained self.module.warn( "Dataflow Deployment already exists and configuration validation and reconciliation " + "is not supported;" + @@ -354,9 +418,9 @@ def process(self): else: # Deployment CRN not found in Tenant, and probably doesn't exist if self.state in ['absent']: + # Deployment not found, and not wanted, return self.module.log( "Dataflow Deployment not found in CDP Tenant %s" % self.dep_crn) - # Deployment to be created elif self.state in ['present']: # create Deployment if not self.module.check_mode: diff --git a/plugins/modules/df_deployment_info.py b/plugins/modules/df_deployment_info.py index 684f5cf6..e8f9e365 100644 --- a/plugins/modules/df_deployment_info.py +++ b/plugins/modules/df_deployment_info.py @@ -33,17 +33,18 @@ requirements: - cdpy options: - name: + crn: description: - - If a name is provided, that DataFlow Deployment will be described - - Must be the string CRN of the deployment + - If a crn is provided, that DataFlow Deployment will be described + - Must be the string CRN of the deployment object type: str aliases: - dep_crn + - name required: False notes: - - This feature this module is for is in Technical Preview + - The feature this module is for is in Technical Preview extends_documentation_fragment: - cloudera.cloud.cdp_sdk_options - cloudera.cloud.cdp_auth_options @@ -77,7 +78,7 @@ returned: always type: str status: - description: The status of a DataFlow enabled environment. + description: The status of a DataFlow deployment. returned: always type: dict contains: @@ -107,11 +108,11 @@ returned: always type: str cloudProvider: - description: the cloud provider for the parent environment. + description: The cloud provider for the parent environment. returned: always type: str region: - description: the region within the parent environment cloud provider. + description: The region within the parent environment cloud provider. returned: always type: str environmentCrn: @@ -167,7 +168,7 @@ returned: always type: bool autoscaleMinNodes: - description: The minimum number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is true. + description: The minimum number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is true. returned: always type: int activeWarningAlertCount: @@ -179,7 +180,7 @@ returned: always type: int staticNodeCount: - description: The static number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is false. + description: The static number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is false. returned: always type: int dfxLocalUrl: diff --git a/plugins/modules/df_readyflow.py b/plugins/modules/df_readyflow.py index 4d143c04..79db0064 100644 --- a/plugins/modules/df_readyflow.py +++ b/plugins/modules/df_readyflow.py @@ -35,9 +35,9 @@ options: name: description: - - If a name is provided, that DataFlow ReadyFlow Definition will be described + - The name of the ReadyFlow to be acted upon. type: str - required: False + required: True state: description: - The declarative state of the ReadyFlow @@ -71,11 +71,10 @@ RETURN = r''' --- readyflow: - description: The listing of ReadyFlow Definitions in the DataFlow Catalog in this CDP Tenant - type: list - returned: always + description: The ReadyFlow Definition + type: dict elements: complex - contains: + returned: always readyflowCrn: description: - The DataFlow readyflow Definition's CRN. @@ -85,7 +84,7 @@ readyflow: description: The details of the ReadyFlow object type: dict - returned: always + returned: varies elements: complex contains: readyflowCrn: @@ -135,11 +134,11 @@ returned: always type: str imported: - description: Whether the ready flow has been imported into the current account. + description: Whether the ready flow has been imported into the current account. returned: always type: bool modifiedTimestamp: - description: THe timestamp the entry was last modified. + description: The timestamp the entry was last modified. returned: always type: int versions: diff --git a/plugins/modules/df_readyflow_info.py b/plugins/modules/df_readyflow_info.py index 5aa414b0..42d5b05a 100644 --- a/plugins/modules/df_readyflow_info.py +++ b/plugins/modules/df_readyflow_info.py @@ -73,7 +73,7 @@ elements: complex contains: readyflowCrn: - description: + description: - The DataFlow readyflow Definition's CRN. - Use this readyflowCrn to address this object returned: always @@ -131,11 +131,11 @@ returned: always type: str imported: - description: Whether the ready flow has been imported into the current account. + description: Whether the ready flow has been imported into the current account. returned: always - type: bool + type: bool modifiedTimestamp: - description: THe timestamp the entry was last modified. + description: The timestamp the entry was last modified. returned: always type: int versions: diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 64526653..31d75418 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -33,13 +33,21 @@ requirements: - cdpy options: - crn: - description: The name or crn of the CDP Environment to host the Dataflow Service + env_crn: + description: + - The CRN of the CDP Environment to host the Dataflow Service + - Required when state is present type: str - required: True + required: Conditional aliases: - name - - env_crn + - crn + df_crn: + description: + - The CRN of the DataFlow Service, if available + - Required when state is absent + type: str + required: Conditional state: description: - The declarative state of the Dataflow Service @@ -48,9 +56,7 @@ default: present choices: - present - - enabled - absent - - disabled nodes_min: description: The minimum number of kubernetes nodes needed for the environment. Note that the lowest minimum is 3 nodes. From 3a3182630ac23defe13a2133d0b070d8cca52442 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 24 Mar 2022 20:03:30 +0000 Subject: [PATCH 6/7] CDF Updates per reviews Improved application deployment playbook to only attempt Kafka flow deployment when Kafka Datahub is found Improved cdpy CRN validation by moving substring definitions to a constant and simplifying the validation logic Improved cdpy.df.describe_service to only run if the dataflow CRN is correctly resolved Force renamed readyflowCRN for an imported ReadyFlow to be addedReadyflowCrn to improve usability Renamed some tasks to improve distinction between similar but different activities like enabling the DF service vs deploying a DF flow Corrected various minor documentation points for DF modules Normalized responses to readyflow_info, deployment_info, and customflow_info to use listings of the full description of objects to simplify user experience Signed-off-by: Daniel Chaffelson --- plugins/modules/df_customflow_info.py | 26 +++++++++++++++----------- plugins/modules/df_deployment_info.py | 13 +++++-------- plugins/modules/df_readyflow.py | 1 - plugins/modules/df_readyflow_info.py | 18 ++++++++---------- plugins/modules/df_service.py | 2 +- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/plugins/modules/df_customflow_info.py b/plugins/modules/df_customflow_info.py index 84e5d19d..0781b7f7 100644 --- a/plugins/modules/df_customflow_info.py +++ b/plugins/modules/df_customflow_info.py @@ -88,50 +88,54 @@ description: The number of versions uploaded to the catalog. returned: always type: str + artifactType: + description: The type of artifact + type: str + returned: when include_details is False createdTimestamp: description: The created timestamp. - returned: always + returned: when include_details is True type: int author: description: Author of the most recent version. - returned: always + returned: when include_details is True type: str description: description: The artifact description. - returned: always + returned: when include_details is True type: str versions: description: The list of artifactDetail versions. - returned: always + returned: when include_details is True type: array contains: crn: description: The flow version CRN. - returned: always + returned: when include_details is True type: str bucketIdentifier: description: The bucketIdentifier of the flow. - returned: always + returned: when include_details is True type: str author: description: The author of the flow. - returned: always + returned: when include_details is True type: str version: description: The version of the flow. - returned: always + returned: when include_details is True type: int timestamp: description: The timestamp of the flow. - returned: always + returned: when include_details is True type: int deploymentCount: description: The number of deployments of the flow. - returned: always + returned: when include_details is True type: int comments: description: Comments about the flow. - returned: always + returned: when include_details is True type: str sdk_out: description: Returns the captured CDP SDK log. diff --git a/plugins/modules/df_deployment_info.py b/plugins/modules/df_deployment_info.py index e8f9e365..87da7020 100644 --- a/plugins/modules/df_deployment_info.py +++ b/plugins/modules/df_deployment_info.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module: df_deployment_info +module:f_deployment_info short_description: Gather information about CDP DataFlow Deployments description: - Gather information about CDP DataFlow Deployments @@ -80,7 +80,7 @@ status: description: The status of a DataFlow deployment. returned: always - type: dict + type:ict contains: state: description: The state of the Deployment. @@ -97,7 +97,7 @@ service: description: Metadata about the parent DataFlow service. returned: always - type: dict + type:ict contains: crn: description: The crn of the parent service. @@ -122,7 +122,7 @@ updated: description: Timestamp of the last time the deployment was modified. returned: always - type: str + type: int clusterSize: description: The initial size of the deployment. returned: always @@ -222,10 +222,7 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): - if self.name is not None: - self.deployments = [self.cdpy.df.describe_deployment(dep_crn=self.name)] - else: - self.deployments = self.cdpy.df.list_deployments() + self.deployments = self.cdpy.df.list_deployments(dep_crn=self.name, described=True) def main(): diff --git a/plugins/modules/df_readyflow.py b/plugins/modules/df_readyflow.py index 79db0064..98c67055 100644 --- a/plugins/modules/df_readyflow.py +++ b/plugins/modules/df_readyflow.py @@ -192,7 +192,6 @@ def __init__(self, module): # Set variables self.name = self._get_param('name') - self.include_details = self._get_param('include_details') self.state = self._get_param('state') # Initialize internal values diff --git a/plugins/modules/df_readyflow_info.py b/plugins/modules/df_readyflow_info.py index 42d5b05a..7bcf1298 100644 --- a/plugins/modules/df_readyflow_info.py +++ b/plugins/modules/df_readyflow_info.py @@ -61,7 +61,6 @@ # Gather summary information about a specific DataFlow Flow Definition using a name - cloudera.cloud.df_readyflow_info: name: my-flow-name - include_details: False ''' RETURN = r''' @@ -72,11 +71,11 @@ returned: always elements: complex contains: - readyflowCrn: + addedReadyflowCrn: description: - - The DataFlow readyflow Definition's CRN. - - Use this readyflowCrn to address this object - returned: always + - The CRN of this readyflow when it is imported to the CDP Tenant + - Use this readyflowCrn to address this object when doing deployments + returned: when readyflow imported is True type: str readyflow: description: The details of the ReadyFlow object @@ -86,8 +85,9 @@ contains: readyflowCrn: description: - - The general base crn of this ReadyFlow - - Different to the unique readyflowCrn containing a UUID4 + - The CRN of this readyflow in the Control Plane + - Different to the addedReadyflowCrn of the imported readyflow within the CDP Tenant + - Use this readyflowCrn when importing the object to your CDP Tenant returned: always type: str name: @@ -189,7 +189,6 @@ def __init__(self, module): # Set variables self.name = self._get_param('name') - self.include_details = self._get_param('include_details') # Initialize internal values self.listing = [] @@ -203,7 +202,7 @@ def __init__(self, module): @CdpModule._Decorators.process_debug def process(self): self.listing = self.cdpy.df.list_readyflows(name=self.name) - if self.include_details and self.listing: + if self.listing: self.flows = [] for this_readyflow in self.listing: if this_readyflow['imported']: @@ -226,7 +225,6 @@ def main(): module = AnsibleModule( argument_spec=CdpModule.argument_spec( name=dict(required=False, type='str'), - include_details=dict(required=False, type='bool', default=True) ), supports_check_mode=True ) diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 31d75418..cff17a35 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -38,7 +38,7 @@ - The CRN of the CDP Environment to host the Dataflow Service - Required when state is present type: str - required: Conditional + required: False aliases: - name - crn From d3201f2ceeb1bce9b9e8e1f34850b0a49ae29cf3 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Fri, 1 Apr 2022 16:19:20 +0100 Subject: [PATCH 7/7] Correct module name in df_deployment_info.py Signed-off-by: Daniel Chaffelson --- plugins/modules/df_deployment_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/df_deployment_info.py b/plugins/modules/df_deployment_info.py index 87da7020..8ef0d835 100644 --- a/plugins/modules/df_deployment_info.py +++ b/plugins/modules/df_deployment_info.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module:f_deployment_info +module: df_deployment_info short_description: Gather information about CDP DataFlow Deployments description: - Gather information about CDP DataFlow Deployments