Move DFX Beta implementation to GA process (#31)

* Renamed df.py to df_service.py to reflect that other DataFlow modules for Deployments will be needed in future
* Include changes to submission parameters and method names for 'cdp df' calls in cdpy
* Modify enable and disable service logic to match new methods and parameters
* df_info now supports name, df_crn, or env_crn for filtering results
* env_info once again checks for DataFlow during detailed Environment information collection
* Added input validation for DataFlow service creation

Signed-off-by: Daniel Chaffelson <[email protected]>

Author: Chaffelson

docs/_sources/df.rst.txt

@@ -181,7 +181,7 @@ Examples
   # Note: These examples do not set authentication details.
 
   # Create a Dataflow Service
-  - cloudera.cloud.df:
+  - cloudera.cloud.df_service:
       name: my-service
       nodes_min: 3
       nodes_max: 10
@@ -191,7 +191,7 @@ Examples
       wait: yes
 
   # Remove a Dataflow Service with Async wait
-  - cloudera.cloud.df:
+  - cloudera.cloud.df_service:
       name: my-service
       persist: False
       state: absent

docsrc/df.rst

@@ -181,7 +181,7 @@ Examples
   # Note: These examples do not set authentication details.
 
   # Create a Dataflow Service
-  - cloudera.cloud.df:
+  - cloudera.cloud.df_service:
       name: my-service
       nodes_min: 3
       nodes_max: 10
@@ -191,7 +191,7 @@ Examples
       wait: yes
 
   # Remove a Dataflow Service with Async wait
-  - cloudera.cloud.df:
+  - cloudera.cloud.df_service:
       name: my-service
       persist: False
       state: absent

plugins/README.md

@@ -16,7 +16,7 @@ modules employ the underlying SDK contained within the `cdpy` Python package.
 | [datalake](./modules/datalake.py) | Create, manage, and destroy CDP Datalakes |
 | [datalake_info](./modules/datalake_info.py) | Gather information about CDP Datalakes |
 | [datalake_runtime_info](./modules/datalake_runtime_info.py) | Gather information about CDP Datalake Runtimes |
-| [df](./modules/df.py) | Enable or disable CDP DataFlow services |
+| [df](./modules/df_service.py) | Enable or disable CDP DataFlow services |
 | [df_info](./modules/df_info.py) | Gather information about CDP DataFlow services |
 | [dw_cluster](./modules/dw_cluster.py) | Create, manage, and destroy CDP Data Warehouse experiences |
 | [dw_cluster_info](./modules/dw_cluster_info.py) | Gather information about CDP Data Warehouse experiences |

plugins/modules/df_info.py

@@ -36,12 +36,24 @@
 options:
   name:
     description:
-      - If a name is provided, that DataFlow Service will be described.
-      - Must be CDP Environment CRN or string name of DataFlow Service
+      - If a name is provided, that DataFlow Service will be described
+      - Must be the string name of the CDP Environment
+      - Mutually exclusive with df_crn and env_crn 
     type: str
     required: False
-    aliases:
-      - crn
+  df_crn:
+    description:
+      - If a df_crn is provided, that DataFlow Service will be described
+      - Mutually exclusive with name and env_crn 
+    type: str
+    required: False
+  env_crn:
+    description:
+      - If an env_crn is provided, the DataFlow Service for that Environment will be described
+      - Mutually exclusive with name and df_crn
+    type: str
+    required: False
+
 notes:
   - This feature this module is for is in Technical Preview
 extends_documentation_fragment:
@@ -59,9 +71,13 @@
 - cloudera.cloud.df_info:
     name: example-service
 
-# Gather detailed information about a named DataFlow Service using a CRN
+# Gather detailed information about a named DataFlow Service using a Dataflow CRN
 - cloudera.cloud.df_info:
-    crn: example-service-crn
+    df_crn: crn:cdp:df:region:tenant-uuid4:service:service-uuid4
+
+# Gather detailed information about a named DataFlow Service using an Environment CRN
+- cloudera.cloud.df_info:
+    df_crn: crn:cdp:environments:region:tenant-uuid4:environment:environment-uuid4
 '''
 
 RETURN = r'''
@@ -159,32 +175,38 @@ def __init__(self, module):
 
         # Set variables
         self.name = self._get_param('name')
+        self.df_crn = self._get_param('df_crn')
+        self.env_crn = self._get_param('env_crn')
 
         # Initialize return values
         self.services = []
 
+        # Initialize internal values
+        self.all_services = []
+
         # Execute logic process
         self.process()
 
     @CdpModule._Decorators.process_debug
     def process(self):
-        if self.name:  # Note that both None and '' will trigger this
-            if self.name.startswith('crn:'):
-                service_single = self.cdpy.df.describe_environment(env_crn=self.name)
-                if service_single is not None:
-                    self.services.append(service_single)
-            else:
-                self.services = self.cdpy.df.list_environments(name=self.name)
+        # Note that parameters are defaulted to None, and are skipped if None at submission
+        self.all_services = self.cdpy.df.list_services(df_crn=self.df_crn, name=self.name, env_crn=self.env_crn)
+        if any(x is not None for x in [self.name, self.df_crn, self.env_crn]):
+            # Any set parameter indicates a describe is preferred to the lower information list command
+            self.services = [self.cdpy.df.describe_service(df_crn=x['crn']) for x in self.all_services]
         else:
-            self.services = self.cdpy.df.list_environments()
+            self.services = self.all_services
 
 
 def main():
     module = AnsibleModule(
         argument_spec=CdpModule.argument_spec(
-            name=dict(required=False, type='str', aliases=['crn']),
+            name=dict(required=False, type='str'),
+            df_crn=dict(required=False, type='str'),
+            env_crn=dict(required=False, type='str'),
         ),
         supports_check_mode=True,
+        mutually_exclusive=['name', 'df_crn', 'env_crn']
     )
 
     result = DFInfo(module)

plugins/modules/df.py renamed to plugins/modules/df_service.py

@@ -83,6 +83,11 @@
     type: bool
     required: False
     default: False
+  terminate:
+    description: Whether or  not to terminate all deployments associated with this DataFlow service
+    type: bool
+    required: False
+    default: False
   wait:
     description:
       - Flag to enable internal polling to wait for the Dataflow Service to achieve the declared state.
@@ -119,7 +124,7 @@
 # Note: These examples do not set authentication details.
 
 # Create a Dataflow Service
-- cloudera.cloud.df:
+- cloudera.cloud.df_service:
     name: my-service
     nodes_min: 3
     nodes_max: 10
@@ -129,7 +134,7 @@
     wait: yes
 
 # Remove a Dataflow Service with Async wait
-- cloudera.cloud.df:
+- cloudera.cloud.df_service:
     name: my-service
     persist: False
     state: absent
@@ -234,12 +239,17 @@ def __init__(self, module):
         super(DFService, self).__init__(module)
 
         # Set variables
-        self.name = self._get_param('name')
+        self.env_crn = self._get_param('env_crn')
+        self.df_crn = self._get_param('df_crn')
         self.nodes_min = self._get_param('nodes_min')
         self.nodes_max = self._get_param('nodes_max')
         self.public_loadbalancer = self._get_param('public_loadbalancer')
-        self.ip_ranges = self._get_param('ip_ranges')
+        self.lb_ip_ranges = self._get_param('loadbalancer_ip_ranges')
+        self.kube_ip_ranges = self._get_param('kube_ip_ranges')
+        self.cluster_subnets = self._get_param('cluster_subnets')
+        self.lb_subnets = self._get_param('lb_subnets')
         self.persist = self._get_param('persist')
+        self.terminate = self._get_param('terminate')
         self.force = self._get_param('force')
 
         self.state = self._get_param('state')
@@ -252,16 +262,16 @@ def __init__(self, module):
 
         # Initialize internal values
         self.target = None
-        self.env_crn = None
 
         # Execute logic process
         self.process()
 
     @CdpModule._Decorators.process_debug
     def process(self):
-        self.env_crn = self.cdpy.environments.resolve_environment_crn(self.name)
         if self.env_crn is not None:
-            self.target = self.cdpy.df.describe_environment(env_crn=self.name)
+            self.env_crn = self.cdpy.environments.resolve_environment_crn(self.env_crn)
+        if self.env_crn is not None or self.df_crn is not None:
+            self.target = self.cdpy.df.describe_service(env_crn=self.env_crn, df_crn=self.df_crn)
 
         if self.target is not None:
             # DF Database Entry exists
@@ -283,19 +293,22 @@ def process(self):
             # Environment does not have DF database entry, and probably doesn't exist
             if self.state in ['absent']:
                 self.module.log(
-                    "Dataflow Service %s already disabled in CDP Environment %s" % (self.name, self.env_crn))
+                    "Dataflow Service already disabled in CDP Environment %s" % self.env_crn)
             elif self.state in ['present']:
                 if self.env_crn is None:
                     self.module.fail_json(msg="Could not retrieve CRN for CDP Environment %s" % self.env)
                 else:
                     # create DF Service
                     if not self.module.check_mode:
-                        self.service = self.cdpy.df.enable_environment(
+                        self.service = self.cdpy.df.enable_service(
                             env_crn=self.env_crn,
-                            authorized_ips=self.ip_ranges,
                             min_nodes=self.nodes_min,
                             max_nodes=self.nodes_max,
-                            enable_public_ip=self.public_loadbalancer
+                            enable_public_ip=self.public_loadbalancer,
+                            lb_ips=self.lb_ip_ranges,
+                            kube_ips=self.kube_ip_ranges,
+                            cluster_subnets=self.cluster_subnets,
+                            lb_subnets=self.lb_subnets
                         )
                         if self.wait:
                             self.service = self._wait_for_enabled()
@@ -305,65 +318,77 @@ def process(self):
 
     def _wait_for_enabled(self):
         return self.cdpy.sdk.wait_for_state(
-            describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn),
+            describe_func=self.cdpy.df.describe_service, params=dict(env_crn=self.env_crn),
             field=['status', 'state'], state=self.cdpy.sdk.STARTED_STATES,
             delay=self.delay, timeout=self.timeout
         )
 
     def _disable_df(self):
         # Attempt clean Disable, which also ensures we have tried at least once before we do a forced removal
         if self.target['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES:
-            self.service = self.cdpy.df.disable_environment(
-                env_crn=self.env_crn,
-                persist=self.persist
+            self.service = self.cdpy.df.disable_service(
+                df_crn=self.df_crn,
+                persist=self.persist,
+                terminate=self.terminate
             )
+        else:
+            self.module.warn("Attempting to disable DataFlow Service but state %s not in Removable States %s"
+                             % (self.target['status']['state'], self.cdpy.sdk.REMOVABLE_STATES))
         if self.wait:
             # Wait for Clean Disable, if possible
             self.service = self.cdpy.sdk.wait_for_state(
-                describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn),
+                describe_func=self.cdpy.df.describe_service, params=dict(df_crn=self.df_crn),
                 field=['status', 'state'],
                 state=self.cdpy.sdk.STOPPED_STATES + self.cdpy.sdk.REMOVABLE_STATES + [None],
                 delay=self.delay, timeout=self.timeout, ignore_failures=True
             )
         else:
-            self.service = self.cdpy.df.describe_environment(env_crn=self.name)
+            self.service = self.cdpy.df.describe_service(df_crn=self.df_crn)
         # Check disable result against need for further forced delete action, in case it didn't work first time around
         if self.service is not None:
             if self.service['status']['state'] in self.cdpy.sdk.REMOVABLE_STATES:
                 if self.force:
-                    self.service = self.cdpy.df.delete_environment(
-                        env_crn=self.env_crn
+                    self.service = self.cdpy.df.reset_service(
+                        df_crn=self.df_crn
                     )
                 else:
                     self.module.fail_json(msg="DF Service Disable failed and Force delete not requested")
             if self.wait:
                 self.service = self.cdpy.sdk.wait_for_state(
-                    describe_func=self.cdpy.df.describe_environment, params=dict(env_crn=self.env_crn),
+                    describe_func=self.cdpy.df.describe_service, params=dict(df_crn=self.df_crn),
                     field=None,  # This time we require removal or declare failure
                     delay=self.delay, timeout=self.timeout
                 )
             else:
-                self.service = self.cdpy.df.describe_environment(env_crn=self.name)
+                self.service = self.cdpy.df.describe_service(df_crn=self.df_crn)
 
 
 def main():
     module = AnsibleModule(
         argument_spec=CdpModule.argument_spec(
-            name=dict(required=True, type='str', aliases=['crn', 'env_crn']),
-            nodes_min=dict(required=False, type='int', default=3, aliases=['min_k8s_node_count']),
-            nodes_max=dict(required=False, type='int', default=3, aliases=['max_k8s_node_count']),
-            public_loadbalancer=dict(required=False, type='bool', default=False, aliases=['use_public_load_balancer']),
-            ip_ranges=dict(required=False, type='list', elements='str', default=list(),
-                           aliases=['authorized_ip_ranges']),
-            persist=dict(required=False, type='bool', default=False),
-            state=dict(required=False, type='str', choices=['present', 'absent'],
+            env_crn=dict(type='str'),
+            df_crn=dict(type='str'),
+            nodes_min=dict(type='int', default=3, aliases=['min_k8s_node_count']),
+            nodes_max=dict(type='int', default=3, aliases=['max_k8s_node_count']),
+            public_loadbalancer=dict(type='bool', default=False, aliases=['use_public_load_balancer']),
+            loadbalancer_ip_ranges=dict(type='list', elements='str', default=None),
+            kube_ip_ranges=dict(type='list', elements='str', default=None),
+            cluster_subnets=dict(type='list', elements='str', default=None),
+            loadbalancer_subnets=dict(type='list', elements='str', default=None),
+            persist=dict(type='bool', default=False),
+            terminate=dict(type='bool', default=False),
+            state=dict(type='str', choices=['present', 'absent'],
                        default='present'),
-            force=dict(required=False, type='bool', default=False, aliases=['force_delete']),
-            wait=dict(required=False, type='bool', default=True),
-            delay=dict(required=False, type='int', aliases=['polling_delay'], default=15),
-            timeout=dict(required=False, type='int', aliases=['polling_timeout'], default=3600)
+            force=dict(type='bool', default=False, aliases=['force_delete']),
+            wait=dict(type='bool', default=True),
+            delay=dict(type='int', aliases=['polling_delay'], default=15),
+            timeout=dict(type='int', aliases=['polling_timeout'], default=3600)
         ),
         supports_check_mode=True,
+        required_if=[
+            ('state', 'present', ('env_crn', ), False),
+            ('state', 'absent', ('df_crn', ), False)
+        ]
     )
 
     result = DFService(module)

plugins/modules/env_info.py

@@ -436,7 +436,7 @@ def process(self):
             for this_env in self.environments:
                 df = None
                 # Removing until DF is GA so we are not dependent on Beta functionality
-                # df = self.cdpy.df.describe_environment(this_env['crn'])
+                df = self.cdpy.df.list_services(env_crn=this_env['crn'])
                 this_env['descendants'] = {
                     'datahub': self.cdpy.datahub.describe_all_clusters(this_env['environmentName']),
                     'dw': self.cdpy.dw.gather_clusters(this_env['crn']),