diff --git a/plugins/modules/df_customflow_info.py b/plugins/modules/df_customflow_info.py new file mode 100644 index 00000000..0781b7f7 --- /dev/null +++ b/plugins/modules/df_customflow_info.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_customflow_info +short_description: Gather information about CDP DataFlow CustomFlow Definitions +description: + - Gather information about CDP DataFlow CustomFlow Definitions +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow Flow Definition will be described + type: str + required: False + include_details: + description: + - If set to false, only a summary of each flow is returned + type: bool + required: False + default: True + +notes: + - The feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List summary information about all Custom DataFlow Flow Definitions +- cloudera.cloud.df_customflow_info: + +# Gather summary information about a specific DataFlow Flow Definition using a name +- cloudera.cloud.df_customflow_info: + name: my-flow-name + include_details: False +''' + +RETURN = r''' +--- +flows: + description: The listing of CustomFlow Definitions in the DataFlow Catalog in this CDP Tenant + type: list + returned: always + elements: complex + contains: + crn: + description: The DataFlow Flow Definition's CRN. + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + modifiedTimestamp: + description: The timestamp the entry was last modified. + returned: always + type: int + versionCount: + description: The number of versions uploaded to the catalog. + returned: always + type: str + artifactType: + description: The type of artifact + type: str + returned: when include_details is False + createdTimestamp: + description: The created timestamp. + returned: when include_details is True + type: int + author: + description: Author of the most recent version. + returned: when include_details is True + type: str + description: + description: The artifact description. + returned: when include_details is True + type: str + versions: + description: The list of artifactDetail versions. + returned: when include_details is True + type: array + contains: + crn: + description: The flow version CRN. + returned: when include_details is True + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: when include_details is True + type: str + author: + description: The author of the flow. + returned: when include_details is True + type: str + version: + description: The version of the flow. + returned: when include_details is True + type: int + timestamp: + description: The timestamp of the flow. + returned: when include_details is True + type: int + deploymentCount: + description: The number of deployments of the flow. + returned: when include_details is True + type: int + comments: + description: Comments about the flow. + returned: when include_details is True + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFCustomFlowInfo(CdpModule): + def __init__(self, module): + super(DFCustomFlowInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.include_details = self._get_param('include_details') + + # Initialize internal values + self.listing = [] + + # Initialize return values + self.flows = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_flow_definitions(name=self.name) + if self.include_details: + self.flows = [ + self.cdpy.df.describe_customflow(x['crn']) + for x in self.listing + if x['artifactType'] == 'flow' # ReadyFlow have different fields + ] + else: + self.flows = self.listing + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str'), + include_details=dict(required=False, type='bool', default=True) + ), + supports_check_mode=True + ) + + result = DFCustomFlowInfo(module) + output = dict(changed=False, flows=result.flows) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_deployment.py b/plugins/modules/df_deployment.py new file mode 100644 index 00000000..94159b70 --- /dev/null +++ b/plugins/modules/df_deployment.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} +#TODO: Update docs +DOCUMENTATION = r''' +--- +module: df_deployment +short_description: Enable or Disable CDP DataFlow Services +description: + - Enable or Disable CDP DataFlow Services +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - The name of the Deployed Flow, or Flow to be Deployed + type: str + required: False + dep_crn: + description: + - The crn of the Deployed Flow to be terminated + - Required if Name is not supplied for termination + type: str + required: False + df_crn: + description: + - The crn of the Dataflow Service + - Required if the df_name is not supplied + type: str + required: False + df_name: + description: + - The Name of the Dataflow Service + - Required if the df_crn is not supplied + type: str + required: False + flow_ver_crn: + description: + - The crn of the specific Version of the Flow to be Deployed + - Required for creating a Deployment if flow_name is not supplied + type: str + required: False + flow_name: + description: + - The Name of the Flow to be Deployed + - Required for creating a Deployment if flow_ver_crn is not supplied + type: str + required: False + flow_ver_crn: + description: + - The crn of the specific Version of the Flow to be Deployed + - Required for creating a Deployment if flow_name is not supplied + type: str + required: False + flow_ver: + description: + - The Version number of the Flow to be Deployed + - If not supplied, the latest version available will be Deployed + type: int + required: False + default: newest + size: + description: + - The Size of the Pod for the Flow to be Deployed into + type: str + default: SMALL + options: + - EXTRA_SMALL + - SMALL + - MEDIUM + - LARGE + required: False + static_node_count: + description: + - The number of nodes to build the Pod on if not using Autoscaling + type: int + required: False + default: 1 + autoscale: + description: + - Whether to use autoscaling of pods for this Deployment + type: bool + required: False + default: False + autoscale_nodes_min: + description: + - The minimum number of nodes to use when Autoscaling + type: int + required: False + default: 1 + autoscale_nodes_max: + description: + - The maximum number of nodes to use when Autoscaling + type: int + required: False + default: 3 + nifi_ver: + description: + - The specific version of NiFi to use in the Deployment + type: str + required: False + default: latest + wait: + description: + - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state. + - If set to FALSE, the module will return immediately. + type: bool + required: False + default: True + autostart_flow: + description: + - Whether to automatically start the Flow once Deployment is complete + type: bool + required: False + default: True + parameter_groups: + description: + - Definitions of Parameters to apply to the Deployed Flow + type: array + required: False + kpis: + description: + - Definitions of KPIs to apply to the Deployed Flow + type: list + required: False + delay: + description: + - The internal polling interval (in seconds) while the module waits for the Dataflow Service to achieve the + declared state. + type: int + required: False + default: 15 + aliases: + - polling_delay + timeout: + description: + - The internal polling timeout (in seconds) while the module waits for the Dataflow Service to achieve the + declared state. + type: int + required: False + default: 3600 + aliases: + - polling_timeout +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Deploy a Dataflow with defaults +- cloudera.cloud.df_deployment: + name: my-flow + +# Remove a Dataflow Service with Async wait +- cloudera.cloud.df_deployment: + name: my-flow-name + df_name: my-env-name + state: absent + wait: yes + async: 3600 + poll: 0 + register: __my_teardown_request + +''' + +RETURN = r''' +--- +deployment: + description: The information about the named DataFlow Deployment + type: dict + returned: always + elements: complex + crn: + description: The deployment CRN. + returned: always + type: str + name: + description: The deployment name. + returned: always + type: str + status: + description: The status of a DataFlow enabled environment. + returned: always + type: dict + contains: + detailedState: + description: The detailed state that the deployment is currently in. + returned: always + type: str + message: + description: A status message for the environment. + returned: always + type: str + state: + description: The state that the deployment is currently in + returned: always + type: str + service: + description: Metadata about the DataFlow service. + returned: always + type: dict + contains: + crn: + description: The crn of the DataFlow service. + returned: always + type: str + name: + description: The name of the CDP Environment. + returned: always + type: str + cloudProvider: + description: The cloud provider + returned: always + type: str + region: + description: The region within the cloud provider + returned: always + type: str + environmentCrn: + description: The CDP Environment CRN + returned: always + type: str + updated: + description: Timestamp of the last time the deployment was modified. + returned: always + type: int + clusterSize: + description: The initial size of the deployment. + returned: always + type: str + flowVersionCrn: + description: The deployment's current flow version CRN. + returned: always + type: str + flowCrn: + description: The deployment's current flow CRN. + returned: always + type: int + nifiUrl: + description: The url to open the deployed flow in NiFi. + returned: always + type: str + autoscaleMaxNodes: + description: The maximum number of nodes that the deployment can scale up to, or null if autoscaling is not enabled for this deployment. + returned: always + type: int + flowName: + description: The name of the flow. + returned: always + type: str + flowVersion: + description: The version of the flow. + returned: always + type: int + currentNodeCount: + description: The current node count. + returned: always + type: int + deployedByCrn: + description: The actor CRN of the person who deployed the flow. + returned: always + type: str + deployedByName: + description: The name of the person who deployed the flow. + returned: always + type: str + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool + autoscaleMinNodes: + description: + - The minimum number of nodes that the deployment will allocate. + - May only be specified when autoscalingEnabled is true. + returned: always + type: int + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool + activeWarningAlertCount: + description: Current count of active alerts classified as a warning. + returned: always + type: int + activeErrorAlertCount: + description: Current count of active alerts classified as an error. + returned: always + type: int + staticNodeCount: + description: + - The static number of nodes that the deployment will allocate. + - May only be specified when autoscalingEnabled is false. + returned: always + type: int + dfxLocalUrl: + description: Base URL to the dfx-local instance running this deployment. + returned: always + type: str + lastUpdatedByName: + description: The name of the person who last updated the deployment. + returned: always + type: str + configurationVersion: + description: The version of the configuration for this deployment. + returned: always + type: int +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFDeployment(CdpModule): + def __init__(self, module): + super(DFDeployment, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.dep_crn = self._get_param('dep_crn') + self.df_crn = self._get_param('df_crn') + self.df_name = self._get_param('df_name') + self.flow_ver_crn = self._get_param('flow_ver_crn') + self.flow_name = self._get_param('flow_name') + self.flow_ver = self._get_param('flow_ver') + self.size = self._get_param('size') + self.static_node_count = self._get_param('static_node_count') + self.autoscale_enabled = self._get_param('autoscale_enabled') + self.autoscale_nodes_min = self._get_param('autoscale_nodes_min') + self.autoscale_nodes_max = self._get_param('autoscale_nodes_max') + self.nifi_ver = self._get_param('nifi_ver') + self.autostart_flow = self._get_param('autostart_flow') + self.parameter_groups = self._get_param('parameter_groups') + self.kpis = self._get_param('kpis') + + self.state = self._get_param('state') + self.wait = self._get_param('wait') + self.delay = self._get_param('delay') + self.timeout = self._get_param('timeout') + + # Initialize return values + self.deployment = None + self.changed = False + + # Initialize internal values + self.target = {} + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + # Prepare information + if self.df_crn is None: + self.df_crn = self.cdpy.df.resolve_service_crn_from_name(self.df_name) + if self.df_crn is None: + self.module.fail_json( + msg="Either df_crn must be supplied or resolvable from df_name") + if self.dep_crn is not None: + self.target = self.cdpy.df.describe_deployment(dep_crn=self.dep_crn) + elif self.name is not None and self.df_crn is not None: + self.target = self.cdpy.df.describe_deployment(df_crn=self.df_crn, name=self.name) + if self.target is not None: + self.dep_crn = self.target['crn'] + # Process execution + if self.target is not None: + # DF Deployment exists + if self.state in ['absent']: + # Existing Deployment to be removed + if self.module.check_mode: + self.module.log( + "Check mode enabled, skipping termination of Deployment %s" % self.dep_crn) + self.deployment = self.target + else: + self._terminate_deployment() + elif self.state in ['present']: + # Existing deployment to be retained + self.module.warn( + "Dataflow Deployment already exists and configuration validation and reconciliation " + + "is not supported;" + + "to change a Deployment, explicitly terminate and recreate it or use the UI") + if self.wait: + self.deployment = self._wait_for_deployed() + else: + self.module.fail_json( + msg="State %s is not valid for this module" % self.state) + else: + # Deployment CRN not found in Tenant, and probably doesn't exist + if self.state in ['absent']: + # Deployment not found, and not wanted, return + self.module.log( + "Dataflow Deployment not found in CDP Tenant %s" % self.dep_crn) + elif self.state in ['present']: + # create Deployment + if not self.module.check_mode: + self._create_deployment() + if self.wait: + self.deployment = self._wait_for_deployed() + else: + pass # Check mode can return the described deployment + else: + self.module.fail_json( + msg="State %s is not valid for this module" % self.state) + + def _create_deployment(self): + if self.flow_ver_crn is None: + # flow_name must be populated if flow_ver_crn is None + self.flow_ver_crn = self.cdpy.df.get_version_crn_from_flow_definition(self.flow_name, self.flow_ver) + self.deployment = self.cdpy.df.create_deployment( + df_crn=self.df_crn, + flow_ver_crn=self.flow_ver_crn, + deployment_name=self.name, + size_name=self.size, + static_node_count=self.static_node_count, + autoscale_enabled=self.autoscale_enabled, + autoscale_nodes_min=self.autoscale_nodes_min, + autoscale_nodes_max=self.autoscale_nodes_max, + nifi_ver=self.nifi_ver, + autostart_flow=self.autostart_flow, + parameter_groups=self.parameter_groups, + kpis=self.kpis, + ) + self.changed = True + + def _wait_for_deployed(self): + return self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.df.describe_deployment, + params=dict(dep_crn=self.dep_crn, df_crn=self.df_crn, name=self.name), + field=['status', 'state'], state=self.cdpy.sdk.STARTED_STATES, + delay=self.delay, timeout=self.timeout + ) + + def _terminate_deployment(self): + if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES: + self.deployment = self.cdpy.df.terminate_deployment( + dep_crn=self.dep_crn + ) + self.changed = True + else: + self.module.warn("Attempting to disable DataFlow Deployment but state %s not in Removable States %s" + % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES)) + if self.wait: + self.deployment = self.cdpy.sdk.wait_for_state( + describe_func=self.cdpy.df.describe_deployment, + params=dict(dep_crn=self.dep_crn), field=None, + delay=self.delay, timeout=self.timeout + ) + else: + self.deployment = self.cdpy.df.describe_deployment(dep_crn=self.dep_crn) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(type='str'), + df_crn=dict(type='str', default=None), + df_name=dict(type='str', default=None), + dep_crn=dict(type='str', default=None), + flow_ver_crn=dict(type='str', default=None), + flow_name=dict(type='str', default=None), + flow_ver=dict(type='int', default=None), + size=dict(type='str', + choices=['EXTRA_SMALL', 'SMALL', 'MEDIUM', 'LARGE'], + default='EXTRA_SMALL', aliases=['size_name']), + static_node_count=dict(type='int', default=1), + autoscale=dict(type='bool', default=False, aliases=['autoscale_enabled']), + autoscale_nodes_min=dict(type='int', default=1), + autoscale_nodes_max=dict(type='int', default=3), + nifi_ver=dict(type='str', default=None), + autostart_flow=dict(type='bool', default=True), + parameter_groups=dict(type='list', default=None), + kpis=dict(type='list', default=None), + state=dict(type='str', choices=['present', 'absent'], + default='present'), + wait=dict(type='bool', default=True), + delay=dict(type='int', aliases=['polling_delay'], default=15), + timeout=dict(type='int', aliases=['polling_timeout'], default=3600) + ), + supports_check_mode=True, + required_one_of=[ + ['df_crn', 'df_name'] + ], + required_if=[ + ['state', 'absent', ['dep_crn', 'name'], True], # One of for termination + ['state', 'present', ['flow_ver_crn', 'flow_name'], True], # One of + ['state', 'present', ['name']] + ], + ) + + result = DFDeployment(module) + output = dict(changed=result.changed, deployment=result.deployment) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_deployment_info.py b/plugins/modules/df_deployment_info.py new file mode 100644 index 00000000..8ef0d835 --- /dev/null +++ b/plugins/modules/df_deployment_info.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_deployment_info +short_description: Gather information about CDP DataFlow Deployments +description: + - Gather information about CDP DataFlow Deployments +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + crn: + description: + - If a crn is provided, that DataFlow Deployment will be described + - Must be the string CRN of the deployment object + type: str + aliases: + - dep_crn + - name + required: False + +notes: + - The feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List basic information about all DataFlow Deployments +- cloudera.cloud.df_deployment_info: + +# Gather detailed information about a named DataFlow Deployment using a name +- cloudera.cloud.df_deployment_info: + name: crn:cdp:df:region:tenant-uuid4:deployment:deployment-uuid4/deployment-uuid4 +''' + +RETURN = r''' +--- +deployments: + description: The information about the named DataFlow Deployment or DataFlow Deployments + type: list + returned: always + elements: complex + contains: + crn: + description: The DataFlow Deployment's CRN. + returned: always + type: str + name: + description: The DataFlow Deployment's name. + returned: always + type: str + status: + description: The status of a DataFlow deployment. + returned: always + type:ict + contains: + state: + description: The state of the Deployment. + returned: always + type: str + detailedState: + description: The state of the Deployment. + returned: always + type: str + message: + description: A status message for the Deployment. + returned: always + type: str + service: + description: Metadata about the parent DataFlow service. + returned: always + type:ict + contains: + crn: + description: The crn of the parent service. + returned: always + type: str + name: + description: The name of the parent environment. + returned: always + type: str + cloudProvider: + description: The cloud provider for the parent environment. + returned: always + type: str + region: + description: The region within the parent environment cloud provider. + returned: always + type: str + environmentCrn: + description: The CDP parent Environment CRN. + returned: always + type: str + updated: + description: Timestamp of the last time the deployment was modified. + returned: always + type: int + clusterSize: + description: The initial size of the deployment. + returned: always + type: str + flowVersionCrn: + description: The deployment's current flow version CRN. + returned: always + type: str + flowCrn: + description: The deployment's current flow CRN. + returned: always + type: str + nifiUrl: + description: The url to open the deployed flow in NiFi. + returned: always + type: str + autoscaleMaxNodes: + description: The maximum number of nodes that the deployment can scale up to, or null if autoscaling is not enabled for this deployment. + returned: always + type: complex + flowName: + description: The name of the flow. + returned: always + type: str + flowVersion: + description: The version of the flow. + returned: always + type: int + currentNodeCount: + description: The current node count. + returned: always + type: int + deployedByCrn: + description: The actor CRN of the person who deployed the flow. + returned: always + type: str + deployedByName: + description: The name of the person who deployed the flow. + returned: always + type: complex + autoscalingEnabled: + description: Whether or not to autoscale the deployment. + returned: always + type: bool + autoscaleMinNodes: + description: The minimum number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is true. + returned: always + type: int + activeWarningAlertCount: + description: Current count of active alerts classified as a warning. + returned: always + type: int + activeErrorAlertCount: + description: Current count of active alerts classified as an error. + returned: always + type: int + staticNodeCount: + description: The static number of nodes that the deployment will allocate. May only be specified when autoscalingEnabled is false. + returned: always + type: int + dfxLocalUrl: + description: Base URL to the DFX Local instance running this deployment. + returned: always + type: string + lastUpdatedByName: + description: The name of the person who last updated the deployment. + returned: always + type: string + configurationVersion: + description: The version of the configuration for this deployment. + returned: always + type: int +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFDeploymentInfo(CdpModule): + def __init__(self, module): + super(DFDeploymentInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + + # Initialize return values + self.deployments = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.deployments = self.cdpy.df.list_deployments(dep_crn=self.name, described=True) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str', aliases=['crn', 'dep_crn']) + ), + supports_check_mode=True, + mutually_exclusive=['name', 'df_crn', 'env_crn'] + ) + + result = DFDeploymentInfo(module) + output = dict(changed=False, deployments=result.deployments) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_readyflow.py b/plugins/modules/df_readyflow.py new file mode 100644 index 00000000..98c67055 --- /dev/null +++ b/plugins/modules/df_readyflow.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_readyflow +short_description: Import or Delete ReadyFlows from your CDP Tenant +description: + - Import or Delete ReadyFlows from your CDP Tenant +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - The name of the ReadyFlow to be acted upon. + type: str + required: True + state: + description: + - The declarative state of the ReadyFlow + type: str + required: False + default: present + choices: + - present + - absent + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# Import a ReadyFlow into your CDP Tenant +- cloudera.cloud.df_readyflow: + name: my-readyflow-name + +# Delete an added ReadyFlow from your CDP Tenant +- cloudera.cloud.df_readyflow: + name: my-readyflow-name + state: absent +''' + +RETURN = r''' +--- +readyflow: + description: The ReadyFlow Definition + type: dict + elements: complex + returned: always + readyflowCrn: + description: + - The DataFlow readyflow Definition's CRN. + - Use this readyflowCrn to address this object + returned: always + type: str + readyflow: + description: The details of the ReadyFlow object + type: dict + returned: varies + elements: complex + contains: + readyflowCrn: + description: + - The general base crn of this ReadyFlow + - Different to the unique readyflowCrn containing a UUID4 + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + author: + description: Author of the most recent version. + returned: always + type: str + summary: + description: The ready flow summary (short). + returned: always + type: str + description: + description: The ready flow description (long). + returned: always + type: str + documentationLink: + description: A link to the ready flow documentation. + returned: always + type: str + notes: + description: Optional notes about the ready flow. + returned: always + type: str + source: + description: The ready flow data source. + returned: always + type: str + sourceDataFormat: + description: The ready flow data source format. + returned: always + type: str + destination: + description: The ready flow data destination. + returned: always + type: str + destinationDataFormat: + description: The ready flow data destination format. + returned: always + type: str + imported: + description: Whether the ready flow has been imported into the current account. + returned: always + type: bool + modifiedTimestamp: + description: The timestamp the entry was last modified. + returned: always + type: int + versions: + description: The list of artifactDetail versions. + returned: When imported is True + type: array + contains: + crn: + description: The artifact version CRN. + returned: always + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: always + type: str + author: + description: The author of the artifact. + returned: always + type: str + version: + description: The version of the artifact. + returned: always + type: int + timestamp: + description: The timestamp of the artifact. + returned: always + type: int + deploymentCount: + description: The number of deployments of the artifact. + returned: always + type: int + comments: + description: Comments about the version. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFReadyFlow(CdpModule): + def __init__(self, module): + super(DFReadyFlow, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + self.state = self._get_param('state') + + # Initialize internal values + self.target = None + self.listing = None + + # Initialize return values + self.readyflow = None + self.changed = False + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_readyflows(name=self.name) + if not self.listing: # return is list with one item if name exists, as name is unique + self.module.fail_json( + msg="ReadyFlow with Name %s is not found" % self.name) + else: + self.target = self.listing[0] + if self.target['imported']: # field is bool + if self.state == 'present': + # ReadyFlow is imported and should be left alone + # helpfully return the detailed description + self.readyflow = self.cdpy.df.describe_added_readyflow( + def_crn=self.target['importedArtifactCrn'] + ) + if self.state == 'absent': + if not self.module.check_mode: + # ReadyFlow is imported and should be deleted + self.readyflow = self.cdpy.df.delete_added_readyflow( + def_crn=self.target['importedArtifactCrn'] + ) + self.changed = True + else: + self.module.log( + "Check mode enabled, skipping deletion of %s" % self.name) + else: + if self.state == 'present': + # ReadyFlow should be imported + if not self.module.check_mode: + self.readyflow = self.cdpy.df.import_readyflow( + def_crn=self.target['readyflowCrn'] + ) + self.changed = True + else: + self.module.log( + "Check mode enabled, skipping import of %s" % self.name) + if self.state == 'absent': + # ReadyFlow is not imported and should stay that way + self.module.log( + "ReadyFlow already not imported to CDP Tenant %s" % self.name) + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=True, type='str'), + state=dict(type='str', choices=['present', 'absent'], + default='present'), + ), + supports_check_mode=True + ) + + result = DFReadyFlow(module) + output = dict(changed=result.changed, readyflow=result.readyflow) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_readyflow_info.py b/plugins/modules/df_readyflow_info.py new file mode 100644 index 00000000..7bcf1298 --- /dev/null +++ b/plugins/modules/df_readyflow_info.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.cloudera.cloud.plugins.module_utils.cdp_common import CdpModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: df_readyflow_info +short_description: Gather information about CDP DataFlow CustomFlow Definitions +description: + - Gather information about CDP DataFlow ReadyFlow Definitions +author: + - "Dan Chaffelson (@chaffelson)" +requirements: + - cdpy +options: + name: + description: + - If a name is provided, that DataFlow ReadyFlow Definition will be described + type: str + required: False + include_details: + description: + - If set to false, only a summary of each ReadyFlow Definition is returned + type: bool + required: False + default: True + +notes: + - This feature this module is for is in Technical Preview +extends_documentation_fragment: + - cloudera.cloud.cdp_sdk_options + - cloudera.cloud.cdp_auth_options +''' + +EXAMPLES = r''' +# Note: These examples do not set authentication details. + +# List summary information about all Custom DataFlow ReadyFlow Definitions +- cloudera.cloud.df_readyflow_info: + +# Gather summary information about a specific DataFlow Flow Definition using a name +- cloudera.cloud.df_readyflow_info: + name: my-flow-name +''' + +RETURN = r''' +--- +flows: + description: The listing of ReadyFlow Definitions in the DataFlow Catalog in this CDP Tenant + type: list + returned: always + elements: complex + contains: + addedReadyflowCrn: + description: + - The CRN of this readyflow when it is imported to the CDP Tenant + - Use this readyflowCrn to address this object when doing deployments + returned: when readyflow imported is True + type: str + readyflow: + description: The details of the ReadyFlow object + type: dict + returned: always + elements: complex + contains: + readyflowCrn: + description: + - The CRN of this readyflow in the Control Plane + - Different to the addedReadyflowCrn of the imported readyflow within the CDP Tenant + - Use this readyflowCrn when importing the object to your CDP Tenant + returned: always + type: str + name: + description: The DataFlow Flow Definition's name. + returned: always + type: str + author: + description: Author of the most recent version. + returned: always + type: str + summary: + description: The ready flow summary (short). + returned: always + type: str + description: + description: The ready flow description (long). + returned: always + type: str + documentationLink: + description: A link to the ready flow documentation. + returned: always + type: str + notes: + description: Optional notes about the ready flow. + returned: always + type: str + source: + description: The ready flow data source. + returned: always + type: str + sourceDataFormat: + description: The ready flow data source format. + returned: always + type: str + destination: + description: The ready flow data destination. + returned: always + type: str + destinationDataFormat: + description: The ready flow data destination format. + returned: always + type: str + imported: + description: Whether the ready flow has been imported into the current account. + returned: always + type: bool + modifiedTimestamp: + description: The timestamp the entry was last modified. + returned: always + type: int + versions: + description: The list of artifactDetail versions. + returned: When imported is True + type: array + contains: + crn: + description: The artifact version CRN. + returned: always + type: str + bucketIdentifier: + description: The bucketIdentifier of the flow. + returned: always + type: str + author: + description: The author of the artifact. + returned: always + type: str + version: + description: The version of the artifact. + returned: always + type: int + timestamp: + description: The timestamp of the artifact. + returned: always + type: int + deploymentCount: + description: The number of deployments of the artifact. + returned: always + type: int + comments: + description: Comments about the version. + returned: always + type: str +sdk_out: + description: Returns the captured CDP SDK log. + returned: when supported + type: str +sdk_out_lines: + description: Returns a list of each line of the captured CDP SDK log. + returned: when supported + type: list + elements: str +''' + + +class DFReadyFlowInfo(CdpModule): + def __init__(self, module): + super(DFReadyFlowInfo, self).__init__(module) + + # Set variables + self.name = self._get_param('name') + + # Initialize internal values + self.listing = [] + + # Initialize return values + self.flows = [] + + # Execute logic process + self.process() + + @CdpModule._Decorators.process_debug + def process(self): + self.listing = self.cdpy.df.list_readyflows(name=self.name) + if self.listing: + self.flows = [] + for this_readyflow in self.listing: + if this_readyflow['imported']: + self.flows.append( + self.cdpy.df.describe_added_readyflow( + def_crn=this_readyflow['importedArtifactCrn'] + ) + ) + else: + self.flows.append( + self.cdpy.df.describe_readyflow( + def_crn=this_readyflow['readyflowCrn'] + ) + ) + else: + self.flows = self.listing + + +def main(): + module = AnsibleModule( + argument_spec=CdpModule.argument_spec( + name=dict(required=False, type='str'), + ), + supports_check_mode=True + ) + + result = DFReadyFlowInfo(module) + output = dict(changed=False, flows=result.flows) + + if result.debug: + output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/df_service.py b/plugins/modules/df_service.py index 772b1aec..cff17a35 100644 --- a/plugins/modules/df_service.py +++ b/plugins/modules/df_service.py @@ -33,13 +33,21 @@ requirements: - cdpy options: - crn: - description: The name or crn of the CDP Environment to host the Dataflow Service + env_crn: + description: + - The CRN of the CDP Environment to host the Dataflow Service + - Required when state is present type: str - required: True + required: False aliases: - name - - env_crn + - crn + df_crn: + description: + - The CRN of the DataFlow Service, if available + - Required when state is absent + type: str + required: Conditional state: description: - The declarative state of the Dataflow Service @@ -48,9 +56,7 @@ default: present choices: - present - - enabled - absent - - disabled nodes_min: description: The minimum number of kubernetes nodes needed for the environment. Note that the lowest minimum is 3 nodes. @@ -88,6 +94,10 @@ type: bool required: False default: False + tags: + description: Tags to apply to the DataFlow Service + type: dict + required: False wait: description: - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state. @@ -147,7 +157,7 @@ RETURN = r''' --- -environments: +services: description: The information about the named DataFlow Service or DataFlow Services type: list returned: always @@ -251,6 +261,7 @@ def __init__(self, module): self.persist = self._get_param('persist') self.terminate = self._get_param('terminate') self.force = self._get_param('force') + self.tags = self._get_param('tags') self.state = self._get_param('state') self.wait = self._get_param('wait') @@ -259,6 +270,7 @@ def __init__(self, module): # Initialize return values self.service = {} + self.changed = False # Initialize internal values self.target = None @@ -307,9 +319,11 @@ def process(self): enable_public_ip=self.public_loadbalancer, lb_ips=self.lb_ip_ranges, kube_ips=self.kube_ip_ranges, + # tags=self.tags, # Currently overstrict blocking of values cluster_subnets=self.cluster_subnets, lb_subnets=self.lb_subnets ) + self.changed = True if self.wait: self.service = self._wait_for_enabled() else: @@ -331,6 +345,10 @@ def _disable_df(self): persist=self.persist, terminate=self.terminate ) + self.changed = True + elif self.target['status']['state'] in self.cdpy.sdk.TERMINATION_STATES: + self.module.warn("DataFlow Service is already Disabling, skipping termination request") + pass else: self.module.warn("Attempting to disable DataFlow Service but state %s not in Removable States %s" % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES)) @@ -351,6 +369,7 @@ def _disable_df(self): self.service = self.cdpy.df.reset_service( df_crn=self.df_crn ) + self.changed = True else: self.module.fail_json(msg="DF Service Disable failed and Force delete not requested") if self.wait: @@ -377,6 +396,7 @@ def main(): loadbalancer_subnets=dict(type='list', elements='str', default=None), persist=dict(type='bool', default=False), terminate=dict(type='bool', default=False), + tags=dict(required=False, type='dict', default=None), state=dict(type='str', choices=['present', 'absent'], default='present'), force=dict(type='bool', default=False, aliases=['force_delete']), @@ -392,7 +412,7 @@ def main(): ) result = DFService(module) - output = dict(changed=False, service=result.service) + output = dict(changed=result.changed, service=result.service) if result.debug: output.update(sdk_out=result.log_out, sdk_out_lines=result.log_lines) diff --git a/plugins/modules/df_info.py b/plugins/modules/df_service_info.py similarity index 95% rename from plugins/modules/df_info.py rename to plugins/modules/df_service_info.py index e1a2debc..787727e1 100644 --- a/plugins/modules/df_info.py +++ b/plugins/modules/df_service_info.py @@ -24,7 +24,7 @@ DOCUMENTATION = r''' --- -module: df_info +module: df_service_info short_description: Gather information about CDP DataFlow Services description: - Gather information about CDP DataFlow Services @@ -82,14 +82,18 @@ RETURN = r''' --- -environments: +services: description: The information about the named DataFlow Service or DataFlow Services type: list returned: always elements: complex contains: crn: - description: The DataFlow Service's parent environment CRN. + description: The DataFlow Service's CRN. + returned: always + type: str + environmentCrn: + description: The DataFlow Service's Parent Environment CRN. returned: always type: str name: @@ -169,9 +173,9 @@ ''' -class DFInfo(CdpModule): +class DFServiceInfo(CdpModule): def __init__(self, module): - super(DFInfo, self).__init__(module) + super(DFServiceInfo, self).__init__(module) # Set variables self.name = self._get_param('name') @@ -209,7 +213,7 @@ def main(): mutually_exclusive=['name', 'df_crn', 'env_crn'] ) - result = DFInfo(module) + result = DFServiceInfo(module) output = dict(changed=False, services=result.services) if result.debug: