From bdd1082da0a04c70a67ce6af31f15b4d1ff07fee Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:06:16 -0400 Subject: [PATCH 1/6] Move service result parsing into service_utils Create cluster_utils for result parsing Signed-off-by: Webster Mudge --- plugins/module_utils/cluster_utils.py | 46 +++++++++++++++++++++++++++ plugins/module_utils/cm_utils.py | 25 --------------- plugins/module_utils/service_utils.py | 46 +++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 25 deletions(-) create mode 100644 plugins/module_utils/cluster_utils.py create mode 100644 plugins/module_utils/service_utils.py diff --git a/plugins/module_utils/cluster_utils.py b/plugins/module_utils/cluster_utils.py new file mode 100644 index 00000000..203d4230 --- /dev/null +++ b/plugins/module_utils/cluster_utils.py @@ -0,0 +1,46 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A common functions for Cloudera Manager cluster management +""" + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + _parse_output, +) + +from cm_client import ApiCluster + + +CLUSTER_OUTPUT = [ + "name", + "display_name", + # "full_version", + "maintenance_mode", + "maintenance_owners", + # "services", + # "parcels", + "entity_status", + "uuid", + # "data_context_refs", + "cluster_type", + "tags", +] + + +def parse_cluster_result(cluster: ApiCluster) -> dict: + # Retrieve full_version as version + output = dict(version=cluster.full_version) + output.update(_parse_output(cluster.to_dict(), CLUSTER_OUTPUT)) + return output diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py index 17000e72..980c1231 100644 --- a/plugins/module_utils/cm_utils.py +++ b/plugins/module_utils/cm_utils.py @@ -38,7 +38,6 @@ ApiParcel, ApiRole, ApiRoleConfigGroup, - ApiService, Configuration, ) from cm_client.rest import ApiException, RESTClientObject @@ -49,23 +48,6 @@ __credits__ = ["frisch@cloudera.com"] __maintainer__ = ["wmudge@cloudera.com"] - -SERVICE_OUTPUT = [ - "client_config_staleness_status", - # "cluster_ref", - "config_staleness_status", - "display_name", - "health_checks", - "health_summary", - "maintenance_mode", - "maintenance_owners", - "name", - "service_state", - "service_version", - "tags", - "type", -] - ROLE_OUTPUT = [ "commission_state", "config_staleness_status", @@ -114,13 +96,6 @@ def _parse_output(entity: dict, filter: list) -> dict: return output -def parse_service_result(service: ApiService) -> dict: - # Retrieve only the cluster_name - output = dict(cluster_name=service.cluster_ref.cluster_name) - output.update(_parse_output(service.to_dict(), SERVICE_OUTPUT)) - return output - - def parse_role_result(role: ApiRole) -> dict: # Retrieve only the host_id, role_config_group, and service identifiers output = dict( diff --git a/plugins/module_utils/service_utils.py b/plugins/module_utils/service_utils.py new file mode 100644 index 00000000..f77f3278 --- /dev/null +++ b/plugins/module_utils/service_utils.py @@ -0,0 +1,46 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A common functions for Cloudera Manager service management +""" + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + _parse_output, +) + +from cm_client import ApiService + +SERVICE_OUTPUT = [ + "client_config_staleness_status", + # "cluster_ref", + "config_staleness_status", + "display_name", + "health_checks", + "health_summary", + "maintenance_mode", + "maintenance_owners", + "name", + "service_state", + "service_version", + "tags", + "type", +] + + +def parse_service_result(service: ApiService) -> dict: + # Retrieve only the cluster_name + output = dict(cluster_name=service.cluster_ref.cluster_name) + output.update(_parse_output(service.to_dict(), SERVICE_OUTPUT)) + return output From 4b38beb962a016f6a91f56a2c0062cc9727575d0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:06:17 -0400 Subject: [PATCH 2/6] Update to use cluster result parsing Update API Signed-off-by: Webster Mudge --- plugins/modules/cluster_info.py | 58 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/plugins/modules/cluster_info.py b/plugins/modules/cluster_info.py index c22b1904..a397e8b4 100644 --- a/plugins/modules/cluster_info.py +++ b/plugins/modules/cluster_info.py @@ -13,9 +13,15 @@ # limitations under the License. from ansible.module_utils.basic import to_native + from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerModule, ) + +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + parse_cluster_result, +) + from cm_client.rest import ApiException from cm_client import ClustersResourceApi @@ -26,7 +32,6 @@ } DOCUMENTATION = r""" ---- module: cluster_info short_description: Retrieve details about one or more clusters description: @@ -45,7 +50,6 @@ """ EXAMPLES = r""" ---- - name: Get information about the cluster cloudera.cluster.cluster_info: host: example.cloudera.com @@ -64,54 +68,45 @@ """ RETURN = r""" ---- clusters: - description: Details about Cloudera Manager Cluster + description: Details about a Cloudera Manager cluster or clusters type: list elements: dict contains: - cluster_type: - description: The type of Cloudera Manager cluster. - type: str - returned: always - cluster_url: - description: Url of Cloudera Manager cluster. + name: + description: The name of the cluster. type: str returned: always display_name: - description: The name of the cluster displayed on the site. + description: The name of the cluster displayed in the Cloudera Manager UI. type: str returned: always entity_status: description: Health status of the cluster. type: str returned: always - full_version: + version: description: Version of the cluster installed. type: str returned: always - hosts_url: - description: Url of all the hosts on which cluster is installed. - type: str - returned: always maintenance_mode: - description: Maintance mode of Cloudera Manager Cluster. + description: Maintance mode of cluster. type: bool returned: always maintenance_owners: - description: List of Maintance owners for Cloudera Manager Cluster. + description: List of maintance owners for cluster. type: list returned: always - name: - description: The name of the cluster. + cluster_type: + description: The type of cluster. type: str returned: always tags: - description: List of tags for Cloudera Manager Cluster. + description: List of tags for cluster. type: list returned: always uuid: - description: Unique ID of the cluster + description: The unique ID of the cluster. type: bool returned: always """ @@ -130,21 +125,26 @@ def process(self): cluster_api_instance = ClustersResourceApi(self.api_client) if self.name: self.output = [ - cluster_api_instance.read_cluster(cluster_name=self.name).to_dict() + parse_cluster_result( + cluster_api_instance.read_cluster(cluster_name=self.name) + ) ] else: - self.output = cluster_api_instance.read_clusters().to_dict()["items"] + self.output = [ + parse_cluster_result(c) + for c in cluster_api_instance.read_clusters().items + ] except ApiException as e: if e.status == 404: pass else: raise e - except KeyError as ke: - self.module.fail_json( - msg="Invalid result object from Cloudera Manager API", - error=to_native(ke), - ) + # except KeyError as ke: + # self.module.fail_json( + # msg="Invalid result object from Cloudera Manager API", + # error=to_native(ke), + # ) def main(): From 2673d85a81255eb9bf77085180585cf7546e6d54 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:06:56 -0400 Subject: [PATCH 3/6] Update API Signed-off-by: Webster Mudge --- plugins/modules/parcel.py | 6 +++--- plugins/modules/parcel_info.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/modules/parcel.py b/plugins/modules/parcel.py index fc537a44..0725988b 100644 --- a/plugins/modules/parcel.py +++ b/plugins/modules/parcel.py @@ -151,7 +151,7 @@ - The state of the parcel. - This shows the progress of state transitions and if there were any errors. type: dict - returned: optional + returned: when supported cluster_name: description: The name of the enclosing cluster. type: dict @@ -159,11 +159,11 @@ display_name: description: Display name of the parcel. type: str - returned: optional + returned: when supported description: description: Description of the parcel. type: str - returned: optional + returned: when supported """ diff --git a/plugins/modules/parcel_info.py b/plugins/modules/parcel_info.py index ddc9f574..673018ff 100644 --- a/plugins/modules/parcel_info.py +++ b/plugins/modules/parcel_info.py @@ -96,19 +96,19 @@ - The state of the parcel. - Shows the progress of state transitions and reports any errors. type: dict - returned: always - clusterRef: - description: A reference to the enclosing cluster. + returned: when supported + cluster_name: + description: The name of the enclosing cluster. type: dict returned: always displayName: description: Display name of the parcel. type: str - returned: always + returned: when supported description: description: Description of the parcel. type: str - returned: always + returned: when supported """ From 361391f9fdd91e68f14f7fbdcbf342f9c5eb207c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:06:57 -0400 Subject: [PATCH 4/6] Refactor to use common service result parsing Signed-off-by: Webster Mudge --- plugins/modules/service.py | 7 +++---- plugins/modules/service_info.py | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/modules/service.py b/plugins/modules/service.py index e0423b91..d80f4300 100644 --- a/plugins/modules/service.py +++ b/plugins/modules/service.py @@ -16,9 +16,11 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerMutableModule, - parse_service_result, resolve_tag_updates, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( + parse_service_result, +) from cm_client import ( ApiEntityTag, @@ -36,7 +38,6 @@ } DOCUMENTATION = r""" ---- module: service short_description: Manage a service in cluster description: @@ -114,7 +115,6 @@ """ EXAMPLES = r""" ---- - name: Establish a cluster service cloudera.cluster.service: host: example.cloudera.com @@ -196,7 +196,6 @@ """ RETURN = r""" ---- service: description: Details about the service. type: dict diff --git a/plugins/modules/service_info.py b/plugins/modules/service_info.py index b2648a80..5ca2858f 100644 --- a/plugins/modules/service_info.py +++ b/plugins/modules/service_info.py @@ -16,12 +16,15 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( ClouderaManagerModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( parse_service_result, ) from cm_client import ServicesResourceApi from cm_client.rest import ApiException + ANSIBLE_METADATA = { "metadata_version": "1.1", "status": ["preview"], From 58d00e5afe408eaed55f3d9591f9a58266ae6e48 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:06:57 -0400 Subject: [PATCH 5/6] Add test option for proxy Signed-off-by: Webster Mudge --- tests/unit/plugins/modules/cluster_info/test_cluster_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/plugins/modules/cluster_info/test_cluster_info.py b/tests/unit/plugins/modules/cluster_info/test_cluster_info.py index 98ffb8ed..184f7085 100644 --- a/tests/unit/plugins/modules/cluster_info/test_cluster_info.py +++ b/tests/unit/plugins/modules/cluster_info/test_cluster_info.py @@ -41,8 +41,8 @@ def conn(): if os.getenv("CM_ENDPOINT", None): conn.update(url=os.getenv("CM_ENDPOINT")) - # if os.getenv("CM_PROXY", None): - # conn.update(proxy=os.getenv("CM_PROXY")) + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) return { **conn, From 1144920e9398aac07234274b909ce2c21f46ced4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 12:07:59 -0400 Subject: [PATCH 6/6] Update cluster API Add require_one_of for role groups Signed-off-by: Webster Mudge --- plugins/modules/cluster.py | 622 ++++++++++++++++++++++++++++++++++--- 1 file changed, 571 insertions(+), 51 deletions(-) diff --git a/plugins/modules/cluster.py b/plugins/modules/cluster.py index af811078..42689d56 100644 --- a/plugins/modules/cluster.py +++ b/plugins/modules/cluster.py @@ -25,6 +25,10 @@ Parcel, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + parse_cluster_result, +) + from cm_client import ( ApiCluster, ApiClusterList, @@ -67,53 +71,568 @@ - Enables cluster management, cluster creation, deletion, and unified control of all services of a cluster. - Create or delete cluster in Cloudera Manager. - Start or stop all services inside the cluster. - - If provided the C(template) parameter, the module will create a cluster based on the template. + - If provided the C(template) parameter, the module will populate its parameters using the cluster template contents. + - Any other parameters will merge on top of the parameters declared by the cluster template. author: - "Ronald Suplina (@rsuplina)" - "Webster Mudge (@wmudge)" requirements: - - cm_client + - cm-client +options: + name: + description: + - The cluster name. + type: str + required: yes + aliases: + - cluster_name + cluster_version: + description: + - The runtime version of the cluster. + type: str + required: no + type: + description: + - The type of cluster. + type: str + required: no + choices: + - BASE_CLUSTER + - COMPUTE_CLUSTER + - EXPERIENCE_CLUSTER + state: + description: + - The state of the cluster. + - If I(state=absent), the cluster's services will be shut down first, then the cluster deleted. + type: str + required: no + default: present + choices: + - present + - absent + - stopped + - started + - restarted + template: + description: + - A cluster template file to populate the module parameters, e.g. as a baseline. + type: path + required: no + aliases: + - cluster_template + add_repositories: + description: + - Flag to add any repositories found in a cluster template should be added to Cloudera Manager. + - Valid only if I(template) is set. + type: bool + required: no + default: no + maintenance: + description: + - Flag enabling the maintenance mode for the cluster. + type: bool + required: no + aliases: + - maintenance_enabled + services: + description: + - List of services and service configurations to enable for the cluster. + type: list + elements: dict + required: no + suboptions: + name: + description: + - The name or reference of the service. + - If not set, a name for the service will be auto-generated. + type: str + required: no + aliases: + - ref + - ref_name + type: + description: + - The type of service to enable. + type: str + required: yes + version: + description: + - The version of the service to enable. + type: str + required: no + config: + description: + - The service configuration values, i.e. the "service-wide" configuration. + type: dict + required: no + role_groups: + description: + - List of role configuration groups for a role type of the service. + - Both base and custom role configuration groups are supported. + type: list + elements: dict + required: no + aliases: + - role_config_groups + suboptions: + name: + description: + - The name of the role group. + - If not set, the base role group, i.e. the default role group, for the role type is used. + type: str + required: no + aliases: + - ref + - ref_name + type: + description: + - The role type for the role group. + type: str + required: yes + aliases: + - role_type + display_name: + description: + - The display name of the role group in the Cloudera Manager UI. + - If not set, the I(name) will be used. + type: str + required: no + config: + description: + - The role group configuration values. + type: dict + required: no + display_name: + description: + - The display name of the service in the Cloudera Manager UI. + - If not set, the I(name) will be used. + type: str + required: no + tags: + description: + - The tags labeling the service. + type: dict + required: no + hosts: + description: + - List of hosts and their configuration attached to the cluster. + type: list + elements: dict + required: no + suboptions: + name: + description: + - The hostname or host ID of the host. + type: str + required: yes + aliases: + - host_id + - hostname + config: + description: + - The host configuration values. + type: dict + required: no + host_template: + description: + - The host template to apply to the host. + type: str + required: no + role_groups: + description: + - List of role groups to associate directly with the host. + type: list + elements: dict + aliases: + - role_config_groups + suboptions: + name: + description: + - The name of the custom role group. + - Mutually exclusive with I(type). + type: str + required: no + aliases: + - ref + - ref_name + service: + description: + - The name of the service associated with the role group. + type: str + required: yes + aliases: + - service_name + - service_ref + type: + description: + - The role type of the base role group for the service. + - Mutually exclusive with I(name). + type: str + required: no + aliases: + - role_type + roles: + description: + - List of per-host role instance configuration overrides. + type: list + elements: dict + required: no + suboptions: + service: + description: + - The name of the service of the role instance. + type: str + required: yes + aliases: + - service_name + - service_ref + type: + description: + - The role type of the role instance. + type: str + required: yes + aliases: + - role_type + config: + description: + - The role instance override configuration values. + type: dict + required: yes + tags: + description: + - The tags labeling the host. + type: dict + required: no + host_templates: + description: + - List of host template definitions for the cluster for use with hosts. + type: list + elements: dict + required: no + suboptions: + name: + description: + - The name of the host template. + type: str + required: yes + role_groups: + description: + - List of role groups, base and custom, to associated with the host template. + type: list + elements: dict + aliases: + - role_config_groups + suboptions: + name: + description: + - The name of the custom role group. + - Mutually exclusive with I(type). + type: str + required: no + aliases: + - ref + - ref_name + service: + description: + - The name of the service associated with the role group. + type: str + required: yes + aliases: + - service_name + - service_ref + type: + description: + - The role type of the base role group for the service. + - Mutually exclusive with I(name). + type: str + required: no + aliases: + - role_type + parcels: + description: + - The parcels by version enabled for a cluster. + - The name of the parcel is the c(key), the version of the parcel is the c(value). + type: dict + required: no + tags: + description: + - The tags labeling the cluster. + type: dict + required: no + display_name: + description: + - The name of the cluster in the Cloudera Manager UI. + - If not set, the I(name) of the cluster will be used. + type: str + required: no + contexts: + description: + - List of data contexts for compute-type clusters. + - Required if I(type=COMPUTE_CLUSTER). + type: list + elements: str + required: no + aliases: + - data_contexts + tls: + description: + - Flag enabling TLS for the cluster. + type: bool + required: no + aliases: + - tls_enabled + - cluster_tls + auto_assign: + description: + - Flag enabling the auto-assignment of role in the cluster. + - This function honors existing or declared assignments. + type: bool + required: no + default: no + aliases: + - auto_assign_roles +extends_documentation_fragment: + - ansible.builtin.action_common_attributes + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint + - cloudera.cluster.purge + - cloudera.cluster.message +attributes: + check_mode: + support: full + diff_mode: + support: full + platform: + platforms: all """ EXAMPLES = r""" -- name: Create an ECS cluster +- name: Create a minimal cluster (can be used by other modules to establish services, etc.) cloudera.cluster.cluster: host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" port: 7180 - clusterName: example-cluster - cluster_version: "1.5.1-b626.p0.42068229" - cluster_type: EXPERIENCE_CLUSTER + cluster_name: example-cluster + cluster_version: "7.1.9" + cluster_type: BASE_CLUSTER state: present -- name: Create a cluster from a cluster template - cloudera.cluster.cm_cluster: +- name: Start all services on a cluster + cloudera.cluster.cluster: host: example.cloudera.com + port: "7180" username: "jane_smith" password: "S&peR4Ec*re" + name: example-cluster + state: started + +- name: Delete a Cluster + cloudera.cluster.cluster: + host: example.cloudera.com port: "7180" - clusterName: example-cluster - template: "./files/cluster-template.json" - add_repositories: yes + username: "jane_smith" + password: "S&peR4Ec*re" + name: example-cluster + state: absent -- name: Start all services on a cluster +- name: Create a cluster with a direct assignment of a base role group cloudera.cluster.cluster: host: example.cloudera.com port: "7180" username: "jane_smith" password: "S&peR4Ec*re" - clusterName: example-cluster + name: example-host-assignment-base + cluster_version: "7.1.9" + type: BASE_CLUSTER + state: present + services: + - name: ROLE_GROUP_ASSIGNMENT + type: ZOOKEEPER + role_groups: + - type: SERVER + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: DIRECT_ROLE_GROUP_ASSIGNMENT + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 33554432 # 32MB + hosts: + - name: worker-02.cldr.internal + role_groups: + - type: SERVER + service: ROLE_GROUP_ASSIGNMENT + +- name: Create a cluster with a direct assignment of a custom role group + cloudera.cluster.cluster: + host: example.cloudera.com + port: "7180" + username: "jane_smith" + password: "S&peR4Ec*re" + name: example-host-assignment-custom + cluster_version: "7.1.9" + type: BASE_CLUSTER + state: present + services: + - name: ROLE_GROUP_ASSIGNMENT + type: ZOOKEEPER + role_groups: + - type: SERVER + display_name: Server Base Group + config: + zookeeper_server_java_heapsize: 134217728 # 128MB + - name: DIRECT_ROLE_GROUP_ASSIGNMENT + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 33554432 # 32MB + hosts: + - name: worker-02.cldr.internal + role_groups: + - type: SERVER + name: DIRECT_ROLE_GROUP_ASSIGNMENT + +- name: Create a cluster with a per-host override of a role configuration + cloudera.cluster.cluster: + host: example.cloudera.com + port: "7180" + username: "jane_smith" + password: "S&peR4Ec*re" + name: example-host-override + cluster_version: "7.1.9" + type: BASE_CLUSTER + state: present + services: + - name: ZK-EXAMPLE + type: ZOOKEEPER + role_groups: + - name: NON-BASE-SERVER + type: SERVER + display_name: Server Custom Group + config: + zookeeper_server_java_heapsize: 75497472 # 72MB + hosts: + - name: worker-01.cldr.internal + roles: + - service: ZK-EXAMPLE + type: SERVER + config: + zookeeper_server_java_heapsize: 67108864 # 64MB + host_template: Example_Template + host_templates: + - name: Example_Template + role_groups: + - NON-BASE-SERVER + parcels: + CDH: "7.1.9" + +- name: Create and establish a minimal base cluster + cloudera.cluster.cluster: + host: example.cloudera.com + port: "7180" + username: "jane_smith" + password: "S&peR4Ec*re" + name: Basic_Cluster + cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" + type: BASE_CLUSTER state: started + services: + - name: core-settings-0 + type: CORE_SETTINGS + display_name: Core Settings + - name: zookeeper-0 + type: ZOOKEEPER + display_name: Zookeeper + config: + zookeeper_datadir_autocreate: yes + - name: hdfs-0 + type: HDFS + config: + zookeeper_service: zookeeper-0 + core_connector: core-settings-0 + role_groups: + - type: DATANODE + config: + dfs_data_dir_list: /dfs/dn + - type: NAMENODE + config: + dfs_name_dir_list: /dfs/nn + - type: SECONDARYNAMENODE + config: + fs_checkpoint_dir_list: /dfs/snn + - name: yarn-0 + type: YARN + config: + hdfs_service: hdfs-0 + zookeeper_service: zookeeper-0 + role_groups: + - type: RESOURCEMANAGER + config: + yarn_scheduler_maximum_allocation_mb: 4096 + yarn_scheduler_maximum_allocation_vcores: 4 + - type: NODEMANAGER + config: + yarn_nodemanager_resource_memory_mb: 4096 + yarn_nodemanager_resource_cpu_vcores: 4 + yarn_nodemanager_local_dirs: /tmp/nm + yarn_nodemanager_log_dirs: /var/log/nm + - type: GATEWAY + config: + mapred_submit_replication: 3 + mapred_reduce_tasks: 6 + host_templates: + - name: Master1 + role_groups: + - service: HDFS + type: NAMENODE + - service: HDFS + type: SECONDARYNAMENODE + - service: YARN + type: RESOURCEMANAGER + - service: YARN + type: JOBHISTORY + - name: Worker + role_groups: + - service: HDFS + type: DATANODE + - service: YARN + type: NODEMANAGER + - service: ZOOKEEPER + type: SERVER + parcels: + CDH: "7.1.9-1.cdh7.1.9.p0.44702451" + hosts: + - name: master-01.cldr.internal + host_template: Master1 + - name: worker-01.cldr.internal + host_template: Worker + - name: worker-02.cldr.internal + host_template: Worker + - name: worker-03.cldr.internal + host_template: Worker -- name: Delete a Cluster - cloudera.cluster.cm_cluster: +- name: Create a cluster from a cluster template + cloudera.cluster.cluster: host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" port: "7180" + name: example-cluster + template: "./files/cluster-template.json" + add_repositories: yes + +- name: Create an ECS cluster + cloudera.cluster.cluster: + host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - clusterName: example-cluster - state: absent + port: 7180 + cluster_name: example-cluster + cluster_version: "1.5.1-b626.p0.42068229" + cluster_type: EXPERIENCE_CLUSTER + state: present """ RETURN = r""" @@ -121,50 +640,42 @@ description: Details about Cloudera Manager Cluster type: dict contains: - cluster_type: - description: The type of cluster created from template. - type: str - returned: optional - cluster_url: - description: Url of Cloudera Manager cluster. + name: + description: The name of the cluster. type: str - returned: optional + returned: always display_name: - description: The name of the cluster displayed on the site. + description: The name of the cluster displayed in the Cloudera Manager UI. type: str - returned: optional + returned: always entity_status: description: Health status of the cluster. type: str - returned: optional - full_version: + returned: always + version: description: Version of the cluster installed. type: str - returned: optional - hosts_url: - description: Url of all the hosts on which cluster is installed. - type: str - returned: optional + returned: always maintenance_mode: - description: Maintance mode of Cloudera Manager Cluster. + description: Maintance mode of cluster. type: bool - returned: optional + returned: always maintenance_owners: - description: List of Maintance owners for Cloudera Manager Cluster. + description: List of maintance owners for cluster. type: list - returned: optional - name: - description: The name of the cluster created. + returned: always + cluster_type: + description: The type of cluster. type: str - returned: optional + returned: always tags: - description: List of tags for Cloudera Manager Cluster. + description: List of tags for cluster. type: list - returned: optional + returned: always uuid: - description: Unique ID of created cluster + description: The unique ID of the cluster. type: bool - returned: optional + returned: always """ @@ -224,7 +735,6 @@ def process(self): # TODO restart arguments (selective restarts) # TODO rolling restart arguments # TODO rolling upgrade arguments - # TODO cluster object return # Retrieve existing cluster existing = None @@ -355,11 +865,11 @@ def process(self): if refresh: # Retrieve the updated cluster details - self.output = self.cluster_api.read_cluster( - cluster_name=self.name - ).to_dict() + self.output = parse_cluster_result( + self.cluster_api.read_cluster(cluster_name=self.name) + ) elif existing: - self.output = existing.to_dict() + self.output = parse_cluster_result(existing) def wait_for_composite_cmd(self, command_id: str): cmd = self.wait_for_command_state( @@ -816,13 +1326,18 @@ def main(): elements="dict", options=dict( name=dict(aliases=["ref", "ref_name"]), - service=dict(required=True, aliases=["service_name"]), + service=dict( + required=True, aliases=["service_name", "service_ref"] + ), type=dict(aliases=["role_type"]), ), aliases=["role_config_groups"], mutually_exclusive=[ ("name", "type"), ], + required_one_of=[ + ("name", "type"), + ], ), roles=dict( type="list", @@ -850,13 +1365,18 @@ def main(): required=True, options=dict( name=dict(aliases=["ref", "ref_name"]), - service=dict(required=True, aliases=["service_name"]), + service=dict( + required=True, aliases=["service_name", "service_ref"] + ), type=dict(aliases=["role_type"]), ), aliases=["role_config_groups"], mutually_exclusive=[ ("name", "type"), ], + requires_one_of=[ + ("name", "type"), + ], ), ), ),