diff --git a/plugins/module_utils/host_template_utils.py b/plugins/module_utils/host_template_utils.py index c04c5105..0a755a71 100644 --- a/plugins/module_utils/host_template_utils.py +++ b/plugins/module_utils/host_template_utils.py @@ -16,26 +16,37 @@ A common functions for Cloudera Manager host templates """ -HOST_TEMPLATE_OUTPUT = ["name", "cluster_ref", "role_config_group_refs"] - - -def _parse_host_template_output(host_template: dict) -> dict: - result = _parse_output(host_template, HOST_TEMPLATE_OUTPUT) - result["cluster_name"] = result["cluster_ref"]["cluster_name"] - result["role_groups"] = [ - role["role_config_group_name"] for role in result["role_config_group_refs"] - ] - del result["cluster_ref"] - del result["role_config_group_refs"] - return result - - -def _parse_host_templates_output(host_templates: list) -> list: - parsed_templates = [template.to_dict() for template in host_templates] - return [ - _parse_host_template_output(template_dict) for template_dict in parsed_templates +from cm_client import ( + ApiClusterRef, + ApiHostTemplate, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, +) + + +def parse_host_template(host_template: ApiHostTemplate) -> dict: + return dict( + name=host_template.name, + cluster_name=host_template.cluster_ref.cluster_name, + role_config_groups=[ + rcg_ref.role_config_group_name + for rcg_ref in host_template.role_config_group_refs + ], + ) + + +def create_host_template_model( + cluster_name: str, + name: str, + role_config_groups: list[ApiRoleConfigGroup], +) -> ApiHostTemplate: + + rcg_refs = [ + ApiRoleConfigGroupRef(role_config_group_name=r.name) for r in role_config_groups ] - -def _parse_output(host_template: dict, keys: list) -> dict: - return {key: host_template[key] for key in keys if key in host_template} + return ApiHostTemplate( + name=name, + cluster_ref=ApiClusterRef(cluster_name=cluster_name), + role_config_group_refs=rcg_refs, + ) diff --git a/plugins/module_utils/role_utils.py b/plugins/module_utils/role_utils.py index 308fa42b..888fbae9 100644 --- a/plugins/module_utils/role_utils.py +++ b/plugins/module_utils/role_utils.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.common.text.converters import to_native +""" +A common functions for Cloudera Manager roles +""" from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( normalize_output, @@ -29,6 +30,7 @@ ApiConfig, ApiConfigList, ApiEntityTag, + ApiRole, ApiRoleList, ApiRoleConfigGroupRef, ApiRoleNameList, @@ -39,7 +41,6 @@ RolesResourceApi, MgmtRolesResourceApi, ) -from cm_client import ApiRole class RoleException(Exception): diff --git a/plugins/modules/host_template.py b/plugins/modules/host_template.py index 135e4c5f..d106c07b 100644 --- a/plugins/modules/host_template.py +++ b/plugins/modules/host_template.py @@ -1,4 +1,7 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,39 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( - _parse_host_template_output, -) -from cm_client import ( - HostTemplatesResourceApi, - ClustersResourceApi, - ApiHostTemplate, - ApiRoleConfigGroupRef, - ApiClusterRef, - ApiHostTemplateList, -) -from cm_client.rest import ApiException - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" ---- module: host_template -short_description: Configure a host template +short_description: Manage a cluster host template description: - - Creates a new host template or updates an existing one - - The module supports C(check_mode). + - Manage a cluster host template. author: + - "Webster Mudge (@wmudge)" - "Ronald Suplina (@rsuplina)" -requirements: - - cm_client options: cluster: description: @@ -58,89 +36,187 @@ - The name of the host template. type: str required: yes - role_groups: + aliases: + - host_template_name + - host_template + - template + role_config_groups: description: - Names of the role configuration groups associated with the host template. type: list - returned: yes - aliases: - - role_config_groups + elements: dict + required: yes + suboptions: + name: + description: + - The name of the custom role config group for the specified service. + - Mutually exclusive with O(type). + type: str + required: no + service: + description: + - The name of the service of the role config group, base or custom. + type: str + required: yes + aliases: + - service_name + type: + description: + - The name of the role type of the base role config group for the specified service. + - Mutually exclusive with O(name). + type: str + required: no + aliases: + - role_type + purge: + description: + - Flag for whether the declared role config groups should append or overwrite any existing entries. + - To clear all configuration overrides or tags, set O(role_config_groups={}), i.e. an empty dictionary, and set O(purge=True). + type: bool + default: False + state: + description: + - The state of the host template. + type: str + required: no + choices: + - present + - absent + default: present +extends_documentation_fragment: + - ansible.builtin.action_common_attributes + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint attributes: check_mode: support: full diff_mode: support: full + platform: + platforms: all +requirements: + - cm-client +seealso: + - module: cloudera.cluster.host_template_info """ EXAMPLES = r""" ---- -- name: Create host template +- name: Provision a host template with a base role config group assignment cloudera.cluster.host_template host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster: "base_cluster" - name: "MyTemplate" - role_groups: ["kafka-GATEWAY-BASE", "atlas-ATLAS_SERVER-BASE" , "hive_on_tez-GATEWAY-BASE"] + cluster: "example_cluster" + name: "Custom Template" + role_config_groups: + - type: DATANODE + service: hdfs-service-1 -- name: Update host template +- name: Provision a host template with a named (custom) role config group assignment cloudera.cluster.host_template host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster: "base_cluster" - name: "MyTemplate" - role_groups: ["kafka-GATEWAY-BASE", "atlas-ATLAS_SERVER-BASE"] + cluster: "example_cluster" + name: "Custom Template" + role_config_groups: + - name: custom-zk-server + service: zookeeper-service-1 -- name: Remove host template +- name: Update (append) a role config group to a host template cloudera.cluster.host_template host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster: "base_cluster" - name: "MyTemplate" + cluster: "example_cluster" + name: "Custom Template" + role_config_groups: + - type: OZONE_DATANODE + service: ozone-service-2 + +- name: Update (reset) the role config groups of a host template + cloudera.cluster.host_template + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + cluster: "example_cluster" + name: "Custom Template" + role_config_groups: + - type: DATANODE + service: hdfs-service-1 + - type: OZONE_DATANODE + service: ozone-service-2 + purge: yes + +- name: Remove a host template + cloudera.cluster.host_template + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + cluster: "example_cluster" + name: "Custom Template" state: "absent" """ RETURN = r""" ---- host_template: - description: - - Retrieve details about host template. + description: Details regarding the host template. type: dict - elements: dict returned: always contains: name: description: - - The name of the host template + - The name of the host template. type: str returned: always cluster_name: - description: A reference to the enclosing cluster. + description: + - A reference to the enclosing cluster. type: str returned: always - role_groups: + role_config_groups: description: - - The role config groups belonging to this host tempalte. + - The role config groups associated with this host template, by role config group name. type: list + elements: str returned: always """ +from cm_client import ( + HostTemplatesResourceApi, + ClustersResourceApi, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiHostTemplateList, + RoleConfigGroupsResourceApi, +) +from cm_client.rest import ApiException + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + create_host_template_model, + parse_host_template, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) + class ClouderaHostTemplate(ClouderaManagerModule): def __init__(self, module): super(ClouderaHostTemplate, self).__init__(module) # Set the parameters - self.cluster_name = self.get_param("cluster") + self.cluster = self.get_param("cluster") self.name = self.get_param("name") - self.role_groups = self.get_param("role_groups") + self.role_config_groups = self.get_param("role_config_groups") + self.purge = self.get_param("purge") self.state = self.get_param("state") - # Initialize the return value - self.host_template = [] - self.host_template_output = [] + # Initialize the return values + self.output = {} self.changed = False self.diff = {} @@ -149,108 +225,171 @@ def __init__(self, module): @ClouderaManagerModule.handle_process def process(self): - host_temp_api_instance = HostTemplatesResourceApi(self.api_client) try: - ClustersResourceApi(self.api_client).read_cluster(self.cluster_name) + ClustersResourceApi(self.api_client).read_cluster(self.cluster) except ApiException as ex: if ex.status == 404: - self.module.fail_json( - msg="Cluster does not exist: " + self.cluster_name - ) + self.module.fail_json(msg="Cluster does not exist: " + self.cluster) else: raise ex + + host_template_api = HostTemplatesResourceApi(self.api_client) + current = None + try: - self.host_template = host_temp_api_instance.read_host_template( - cluster_name=self.cluster_name, + current = host_template_api.read_host_template( + cluster_name=self.cluster, host_template_name=self.name, ) except ApiException as ex: - if ex.status == 404: - pass - else: + if ex.status != 404: raise ex - if self.host_template: - if self.module._diff: - current = { - item.role_config_group_name - for item in self.host_template.role_config_group_refs - } - incoming = set(self.role_groups) - self.diff.update( - before=list(current - incoming), after=list(incoming - current) - ) + if self.state == "absent": + if current: + self.changed = True + + if self.module._diff: + self.diff = dict(before=parse_host_template(current), after=dict()) - if self.state == "present": - host_template_body = ApiHostTemplate( - cluster_ref=ApiClusterRef( - cluster_name=self.cluster_name, display_name=self.cluster_name - ), - name=self.name, - role_config_group_refs=[ - ApiRoleConfigGroupRef(role_config_group_name=group) - for group in self.role_groups - ], - ) - if self.host_template: if not self.module.check_mode: - host_temp_api_instance.update_host_template( - cluster_name=self.cluster_name, + host_template_api.delete_host_template( + cluster_name=self.cluster, host_template_name=self.name, - body=host_template_body, ) - self.changed = True - else: - body = ApiHostTemplateList(items=[host_template_body]) - if not self.module.check_mode: - host_temp_api_instance.create_host_templates( - cluster_name=self.cluster_name, body=body - ) - self.changed = True + elif self.state == "present": + rcg_api = RoleConfigGroupsResourceApi(self.api_client) - self.host_template_output = _parse_host_template_output( - host_temp_api_instance.read_host_template( - cluster_name=self.cluster_name, - host_template_name=self.name, - ).to_dict() - ) + # Find all of the role config group references + incoming_rcgs = list[ApiRoleConfigGroup]() + for rcg in self.role_config_groups: + if rcg["name"] is not None: + custom_rcg = rcg_api.read_role_config_group( + cluster_name=self.cluster, + service_name=rcg["service"], + role_config_group_name=rcg["name"], + ) + incoming_rcgs.append(custom_rcg) + else: + base_rcg = get_base_role_config_group( + api_client=self.api_client, + cluster_name=self.cluster, + service_name=rcg["service"], + role_type=rcg["type"].upper(), + ) + if base_rcg is None: + self.module.fail_json( + msg=f"Role type '{rcg['type']}' not found for service '{rcg['service']}' in cluster '{self.cluster}'" + ) + incoming_rcgs.append(base_rcg) - if self.state == "absent": - if not self.module.check_mode: - self.host_template_output = _parse_host_template_output( - host_temp_api_instance.delete_host_template( - cluster_name=self.cluster_name, - host_template_name=self.name, - ).to_dict() + # If exists, modify + if current: + # Reconcile host template differences + current_rcg_names = set( + [ + rcg.role_config_group_name + for rcg in current.role_config_group_refs + ] ) + incoming_rcg_names = set([rcg.name for rcg in incoming_rcgs]) + + additions = incoming_rcg_names - current_rcg_names + deletions = set() + + if additions or self.purge: + updated_rcg_names = current_rcg_names | additions + + if self.purge: + deletions = current_rcg_names - incoming_rcg_names + updated_rcg_names = updated_rcg_names - deletions + + if additions or deletions: + self.changed = True + + if self.module._diff: + current_diff = parse_host_template(current) + updated_diff = dict(**current_diff) + updated_diff.role_config_groups = updated_rcg_names + self.diff.update( + before=current_diff, after=dict(updated_diff) + ) + + current.role_config_group_refs = [ + ApiRoleConfigGroupRef(rcg_name) + for rcg_name in updated_rcg_names + ] + + if not self.module.check_mode: + current = host_template_api.update_host_template( + cluster_name=self.cluster, + host_template_name=self.name, + body=current, + ) + # Else, create + else: self.changed = True + created_host_template = create_host_template_model( + cluster_name=self.cluster, + name=self.name, + role_config_groups=incoming_rcgs, + ) + + if self.module._diff: + self.diff.update( + before=dict(), after=parse_host_template(created_host_template) + ) + + if not self.module.check_mode: + current = host_template_api.create_host_templates( + cluster_name=self.cluster, + body=ApiHostTemplateList(items=[created_host_template]), + ).items[0] + + self.output = parse_host_template(current) + else: + self.module.fail_json(msg=f"Invalid state: {self.state}") + def main(): module = ClouderaManagerModule.ansible_module( argument_spec=dict( - cluster=dict(required=True, type="str", aliases=["cluster_name"]), - name=dict(required=True, type="str"), - role_groups=dict( - required=False, type="list", aliases=["role_config_groups"] + cluster=dict(required=True, aliases=["cluster_name"]), + name=dict( + required=True, + aliases=["host_template_name", "host_template", "template"], ), + role_config_groups=dict( + type="list", + elements="dict", + options=dict( + name=dict(), + service=dict(required=True, aliases=["service_name"]), + type=dict(aliases=["role_type"]), + ), + mutually_exclusive=[ + ("name", "type"), + ], + ), + purge=dict(type="bool", default=False), state=dict( type="str", default="present", choices=["present", "absent"], ), ), - supports_check_mode=True, required_if=[ - ("state", "present", ("cluster", "role_groups")), + ("state", "present", ("cluster", "role_config_groups")), ], + supports_check_mode=True, ) result = ClouderaHostTemplate(module) output = dict( changed=result.changed, - host_template_output=result.host_template_output, + host_template=result.output, ) if module._diff: output.update(diff=result.diff) diff --git a/plugins/modules/host_template_info.py b/plugins/modules/host_template_info.py index 3ea828df..a8f9259a 100644 --- a/plugins/modules/host_template_info.py +++ b/plugins/modules/host_template_info.py @@ -1,4 +1,7 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,33 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, -) -from cm_client import HostTemplatesResourceApi, ClustersResourceApi -from cm_client.rest import ApiException -from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( - _parse_host_template_output, - _parse_host_templates_output, -) - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" ---- module: host_template_info -short_description: Retrieve details of host templates. +short_description: Retrieve details regarding a cluster's host templates. description: - - Collects detailed information about individual or all host templates. - - The module supports C(check_mode). + - Collects detailed information about individual or all host templates for a cluster. author: - "Ronald Suplina (@rsuplina)" -requirements: - - cm_client + - "Webster Mudge (@wmudge)" options: cluster: description: @@ -52,31 +36,43 @@ - The name of the host template. type: str required: no +extends_documentation_fragment: + - ansible.builtin.action_common_attributes + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint +attributes: + check_mode: + support: full + diff_mode: + support: full + platform: + platforms: all +requirements: + - cm-client +seealso: + - module: cloudera.cluster.host_template """ EXAMPLES = r""" ---- - name: Retrieve the defailts about a specific host template cloudera.cluster.host_template_info host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster: "cfm_cluster" - name: "cfm_host_template" + cluster: "example_cluster" + name: "example_host_template" - name: Retrieve the details about all host templates within the cluster cloudera.cluster.host_template_info host: example.cloudera.com username: "jane_smith" password: "S&peR4Ec*re" - cluster: "cfm_cluster" + cluster: "example_cluster" """ RETURN = r""" ---- -host_template_info: - description: - - Details about host template. +host_templates: + description: List of details about the host templates. type: list elements: dict returned: always @@ -87,17 +83,31 @@ type: str returned: always cluster_name: - description: A reference to the enclosing cluster. + description: + - A reference to the enclosing cluster. type: dict returned: always - role_groups: + role_config_groups: description: - - The names of the role config groups + - The role config groups associated with this host template, by role config group name. type: list elements: str returned: always """ +from cm_client import ( + HostTemplatesResourceApi, + ClustersResourceApi, +) +from cm_client.rest import ApiException + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + parse_host_template, +) + class ClouderaHostTemplateInfo(ClouderaManagerModule): def __init__(self, module): @@ -108,7 +118,7 @@ def __init__(self, module): self.name = self.get_param("name") # Initialize the return value - self.host_templates_output = [] + self.output = [] # Execute the logic self.process() @@ -125,25 +135,29 @@ def process(self): else: raise ex - host_temp_api_instance = HostTemplatesResourceApi(self.api_client) + host_template_api = HostTemplatesResourceApi(self.api_client) + if self.name: try: - self.host_templates_output = _parse_host_template_output( - host_temp_api_instance.read_host_template( - cluster_name=self.cluster_name, - host_template_name=self.name, - ).to_dict() + self.output.append( + parse_host_template( + host_template_api.read_host_template( + cluster_name=self.cluster_name, + host_template_name=self.name, + ) + ) ) except ApiException as ex: if ex.status != 404: raise ex else: - self.host_templates_output = _parse_host_templates_output( - host_temp_api_instance.read_host_templates( + self.output = [ + parse_host_template(ht) + for ht in host_template_api.read_host_templates( cluster_name=self.cluster_name ).items - ) + ] def main(): @@ -159,7 +173,7 @@ def main(): output = dict( changed=False, - host_templates_output=result.host_templates_output, + host_templates=result.output, ) if result.debug: diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index de4b7a6c..112b5b2f 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -24,6 +24,8 @@ ApiConfig, ApiConfigList, ApiHostRef, + ApiHostTemplate, + ApiHostTemplateList, ApiRole, ApiRoleConfigGroup, ApiRoleConfigGroupList, @@ -36,6 +38,8 @@ ApiServiceState, ClustersResourceApi, CommandsResourceApi, + HostsResourceApi, + HostTemplatesResourceApi, MgmtRolesResourceApi, MgmtRoleCommandsResourceApi, MgmtRoleConfigGroupsResourceApi, @@ -53,6 +57,9 @@ from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( get_host_ref, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + create_host_template_model, +) from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( get_mgmt_roles, provision_service_role, @@ -444,6 +451,44 @@ def deregister_role_config_group( ) +def register_host_template( + api_client: ApiClient, + registry: list[ApiHostTemplate], + cluster: ApiCluster, + host_template: ApiHostTemplate, +) -> ApiHostTemplate: + host_template_api = HostTemplatesResourceApi(api_client) + + # Create the host template + created_host_template = host_template_api.create_host_templates( + cluster_name=cluster.name, body=ApiHostTemplateList(items=[host_template]) + ).items[0] + + # Record the host template + registry.append(created_host_template) + + # Return the provisioned host template + return created_host_template + + +def deregister_host_template( + api_client: ApiClient, + registry: list[ApiHostTemplate], +) -> ApiHostTemplate: + host_template_api = HostTemplatesResourceApi(api_client) + + # Delete the host templates + for ht in registry: + try: + host_template_api.delete_host_template( + cluster_name=ht.cluster_ref.cluster_name, + host_template_name=ht.name, + ) + except ApiException as e: + if e.status != 404: + raise e + + def service_wide_config( api_client: ApiClient, service: ApiService, params: dict, message: str ) -> Generator[ApiService]: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index dd82d8ee..84680ebb 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -37,6 +37,7 @@ ApiCluster, ApiCommand, ApiConfig, + ApiHostTemplate, ApiHostRef, ApiHostRefList, ApiRole, @@ -89,6 +90,8 @@ register_service, deregister_role_config_group, register_role_config_group, + deregister_host_template, + register_host_template, set_cm_role_config, set_cm_role_config_group, set_role_config_group, @@ -1183,3 +1186,27 @@ def _wrapper( deregister_role_config_group( api_client=cm_api_client, registry=role_config_groups, message=message ) + + +@pytest.fixture(scope="function") +def host_template_factory( + cm_api_client, +) -> Generator[Callable[[ApiCluster, ApiHostTemplate], ApiHostTemplate]]: + # Track the created host templates + host_templates = list[ApiHostTemplate]() + + # Yield the host template factory function to the tests + def _wrapper( + cluster: ApiCluster, host_template: ApiHostTemplate + ) -> ApiHostTemplate: + return register_host_template( + api_client=cm_api_client, + registry=host_templates, + cluster=cluster, + host_template=host_template, + ) + + yield _wrapper + + # Delete any registered host templates + deregister_host_template(api_client=cm_api_client, registry=host_templates) diff --git a/tests/unit/plugins/modules/host_template/test_host_template.py b/tests/unit/plugins/modules/host_template/test_host_template.py index f985bbb7..3c391446 100644 --- a/tests/unit/plugins/modules/host_template/test_host_template.py +++ b/tests/unit/plugins/modules/host_template/test_host_template.py @@ -1,4 +1,6 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,90 +19,548 @@ __metaclass__ = type import logging -import os import pytest +from collections.abc import Generator +from pathlib import Path + +from cm_client import ( + ApiHostTemplate, + ApiHostTemplateList, + ApiHostRef, + ApiRole, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiService, + HostTemplatesResourceApi, +) +from cm_client.rest import ApiException + from ansible_collections.cloudera.cluster.plugins.modules import host_template + +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, + deregister_service, + register_service, ) LOG = logging.getLogger(__name__) -@pytest.fixture -def conn(): - conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) +@pytest.fixture(scope="module") +def zookeeper(cm_api_client, base_cluster, request) -> Generator[ApiService]: + # Keep track of the provisioned service(s) + service_registry = list[ApiService]() + + # Get the current cluster hosts + hosts = get_cluster_hosts(cm_api_client, base_cluster) - if os.getenv("CM_HOST", None): - conn.update(host=os.getenv("CM_HOST")) + id = Path(request.node.parent.name).stem - if os.getenv("CM_PORT", None): - conn.update(port=os.getenv("CM_PORT")) + zk_service = ApiService( + name=f"test-zk-{id}", + type="ZOOKEEPER", + display_name=f"ZooKeeper ({id})", + # Add a SERVER role (so we can start the service -- a ZK requirement!) + roles=[ApiRole(type="SERVER", host_ref=ApiHostRef(hosts[0].host_id))], + ) + + # Provision and yield the created service + yield register_service( + api_client=cm_api_client, + registry=service_registry, + cluster=base_cluster, + service=zk_service, + ) + + # Remove the created service + deregister_service(api_client=cm_api_client, registry=service_registry) - if os.getenv("CM_ENDPOINT", None): - conn.update(url=os.getenv("CM_ENDPOINT")) - if os.getenv("CM_PROXY", None): - conn.update(proxy=os.getenv("CM_PROXY")) +@pytest.fixture(autouse=True) +def resettable_host_templates(cm_api_client, base_cluster) -> Generator[None]: + host_template_api = HostTemplatesResourceApi(cm_api_client) - return { - **conn, - "verify_tls": "no", - "debug": "no", + # Get the current state of host templates on the cluster + initial_host_templates = { + ht.name: ht + for ht in host_template_api.read_host_templates( + cluster_name=base_cluster.name, + ).items } + # Yield to tests + yield + + # Reset host templates to initial set + current_host_templates = host_template_api.read_host_templates( + cluster_name=base_cluster.name, + ).items + + # Each current host template + for ht in current_host_templates: + # If new, remove + if ht.name not in initial_host_templates: + host_template_api.delete_host_template( + cluster_name=base_cluster.name, + host_template_name=ht.name, + ) + # Else, update/reset + else: + initial_ht = initial_host_templates.pop(ht.name) + host_template_api.update_host_template( + cluster_name=base_cluster.name, + host_template_name=ht.name, + body=initial_ht, + ) + + # If missing, restore + if initial_host_templates: + host_template_api.create_host_templates( + cluster_name=base_cluster.name, + body=ApiHostTemplateList(items=initial_host_templates.values()), + ) + -def test_create_host_template(module_args, conn): - conn.update( - cluster="cloudera.cluster.example", - name="New_template", - role_groups=["atlas-ATLAS_SERVER-BASE", "atlas-GATEWAY-BASE"], +@pytest.fixture() +def existing_host_template( + cm_api_client, zookeeper, request +) -> Generator[ApiHostTemplate]: + host_template_api = HostTemplatesResourceApi(cm_api_client) + + id = f"pytest-{request.node.name}" + + base_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + created_host_template = host_template_api.create_host_templates( + cluster_name=zookeeper.cluster_ref.cluster_name, + body=ApiHostTemplateList( + items=[ + ApiHostTemplate( + name=id, + role_config_group_refs=[ + ApiRoleConfigGroupRef(role_config_group_name=base_rcg.name), + ], + ) + ] + ), + ).items[0] + + yield created_host_template + + try: + host_template_api.delete_host_template( + cluster_name=zookeeper.cluster_ref.cluster_name, + host_template_name=created_host_template.name, + ) + except ApiException as ex: + if ex.status != 404: + raise ex + + +def test_host_template_missing_cluster(conn, module_args): + module_args( + { + **conn, + "name": "EXAMPLE", + "role_config_groups": [], + } ) - module_args(conn) + with pytest.raises( + AnsibleFailJson, match="missing required arguments: cluster" + ) as e: + host_template.main() + + +def test_host_template_missing_name(conn, module_args): + module_args( + { + **conn, + "cluster": "EXAMPLE", + "role_config_groups": [], + } + ) + + with pytest.raises(AnsibleFailJson, match="missing required arguments: name") as e: + host_template.main() + + +def test_host_template_missing_role_config_groups_on_present(conn, module_args): + module_args( + { + **conn, + "cluster": "EXAMPLE", + "name": "EXAMPLE", + } + ) + + with pytest.raises( + AnsibleFailJson, + match="state is present but all of the following are missing: role_config_groups", + ) as e: + host_template.main() + + +def test_host_template_provision_invalid_cluster(conn, module_args): + module_args( + { + **conn, + "cluster": "INVALID", + "name": "Example", + "role_config_groups": [ + { + "service": "zookeeper", + "type": "SERVER", + } + ], + } + ) + + with pytest.raises(AnsibleFailJson, match="Cluster does not exist: INVALID") as e: + host_template.main() + + +def test_host_template_provision_invalid_base_rcg_service(conn, module_args, zookeeper): + module_args( + { + **conn, + "cluster": zookeeper.cluster_ref.cluster_name, + "name": "Example", + "role_config_groups": [ + { + "service": "INVALID", + "type": "SERVER", + } + ], + } + ) + + with pytest.raises( + AnsibleFailJson, match="Service 'INVALID' not found in cluster" + ) as e: + host_template.main() + + +def test_host_template_provision_invalid_base_rcg_name(conn, module_args, zookeeper): + module_args( + { + **conn, + "cluster": zookeeper.cluster_ref.cluster_name, + "name": "Example", + "role_config_groups": [ + { + "service": zookeeper.name, + "type": "INVALID", + } + ], + } + ) + + with pytest.raises( + AnsibleFailJson, + match=f"Role type 'INVALID' not found for service '{zookeeper.name}'", + ) as e: + host_template.main() + + +def test_host_template_provision_base_rcg( + conn, module_args, cm_api_client, zookeeper, request +): + id = f"pytest-{request.node.name}" + + base_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + module_args( + { + **conn, + "cluster": zookeeper.cluster_ref.cluster_name, + "name": id, + "role_config_groups": [ + { + "service": zookeeper.name, + "type": base_rcg.role_type, + } + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == True + assert e.value.host_template["name"] == id + assert base_rcg.name in e.value.host_template["role_config_groups"] + + # Idempotency with pytest.raises(AnsibleExitJson) as e: host_template.main() - LOG.info(str(e.value.host_template_output)) + assert e.value.changed == False + assert e.value.host_template["name"] == id + assert base_rcg.name in e.value.host_template["role_config_groups"] -def test_update_host_template(module_args, conn): - conn.update( - cluster="cloudera.cluster.example", - name="New_template", - role_groups=[ - "atlas-ATLAS_SERVER-BASE", - "tez-GATEWAY-BASE", - "hdfs-NAMENODE-BASE", - ], +def test_host_template_provision_custom_rcg( + conn, module_args, zookeeper, role_config_group_factory, request +): + id = f"pytest-{request.node.name}" + + custom_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=ApiRoleConfigGroup(name=f"SERVER-{id}", role_type="SERVER"), + ) + + module_args( + { + **conn, + "cluster": zookeeper.cluster_ref.cluster_name, + "name": id, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": custom_rcg.name, + } + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == True + assert e.value.host_template["name"] == id + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + # Idempotency + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == False + assert e.value.host_template["name"] == id + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + +def test_host_template_existing_duplicate_type( + module_args, + conn, + zookeeper, + role_config_group_factory, + existing_host_template, + request, +): + id = f"pytest-{request.node.name}" + + custom_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=ApiRoleConfigGroup(name=f"SERVER-{id}", role_type="SERVER"), + ) + + module_args( + { + **conn, + "cluster": existing_host_template.cluster_ref.cluster_name, + "name": existing_host_template.name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": custom_rcg.name, + } + ], + } + ) + + with pytest.raises( + AnsibleFailJson, + match="The template already contains a role config group for type SERVER", + ): + host_template.main() + + +def test_host_template_existing_add( + module_args, + conn, + cm_api_client, + zookeeper, + role_config_group_factory, + existing_host_template, + request, +): + host_template_api = HostTemplatesResourceApi(cm_api_client) + + id = f"pytest-{request.node.name}" + + custom_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=ApiRoleConfigGroup(name=f"GATEWAY-{id}", role_type="GATEWAY"), + ) + + module_args( + { + **conn, + "cluster": existing_host_template.cluster_ref.cluster_name, + "name": existing_host_template.name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": custom_rcg.name, + } + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == True + assert e.value.host_template["name"] == existing_host_template.name + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + updated_host_template = host_template_api.read_host_template( + cluster_name=existing_host_template.cluster_ref.cluster_name, + host_template_name=existing_host_template.name, + ) + assert set( + [ + rcg_ref.role_config_group_name + for rcg_ref in updated_host_template.role_config_group_refs + ] + ) == set(e.value.host_template["role_config_groups"]) + + # Idempotency + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == False + assert e.value.host_template["name"] == existing_host_template.name + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + updated_host_template = host_template_api.read_host_template( + cluster_name=existing_host_template.cluster_ref.cluster_name, + host_template_name=existing_host_template.name, + ) + assert set( + [ + rcg_ref.role_config_group_name + for rcg_ref in updated_host_template.role_config_group_refs + ] + ) == set(e.value.host_template["role_config_groups"]) + + +def test_host_template_existing_purge( + module_args, + conn, + cm_api_client, + zookeeper, + role_config_group_factory, + existing_host_template, + request, +): + host_template_api = HostTemplatesResourceApi(cm_api_client) + + id = f"pytest-{request.node.name}" + + custom_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=ApiRoleConfigGroup(name=f"SERVER-{id}", role_type="SERVER"), ) - module_args(conn) + module_args( + { + **conn, + "cluster": existing_host_template.cluster_ref.cluster_name, + "name": existing_host_template.name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": custom_rcg.name, + } + ], + "purge": True, + } + ) with pytest.raises(AnsibleExitJson) as e: host_template.main() - LOG.info(str(e.value.host_template_output)) + assert e.value.changed == True + assert e.value.host_template["name"] == existing_host_template.name + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + updated_host_template = host_template_api.read_host_template( + cluster_name=existing_host_template.cluster_ref.cluster_name, + host_template_name=existing_host_template.name, + ) + assert set( + [ + rcg_ref.role_config_group_name + for rcg_ref in updated_host_template.role_config_group_refs + ] + ) == set(e.value.host_template["role_config_groups"]) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host_template.main() -def remove_template(module_args, conn): - conn.update( - cluster="Base-PVC", - name="4", - role_groups=[ - "atlas-ATLAS_SERVER-BASE", - "tez-GATEWAY-BASE", - "hdfs-NAMENODE-BASE", - ], - state="absent", + assert e.value.changed == False + assert e.value.host_template["name"] == existing_host_template.name + assert custom_rcg.name in e.value.host_template["role_config_groups"] + + updated_host_template = host_template_api.read_host_template( + cluster_name=existing_host_template.cluster_ref.cluster_name, + host_template_name=existing_host_template.name, ) + assert set( + [ + rcg_ref.role_config_group_name + for rcg_ref in updated_host_template.role_config_group_refs + ] + ) == set(e.value.host_template["role_config_groups"]) + + +def test_host_template_state_absent( + conn, module_args, cm_api_client, existing_host_template +): + host_template_api = HostTemplatesResourceApi(cm_api_client) + + module_args( + { + **conn, + "cluster": existing_host_template.cluster_ref.cluster_name, + "name": existing_host_template.name, + "state": "absent", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_template.main() + + assert e.value.changed == True - module_args(conn) + # Idempotency with pytest.raises(AnsibleExitJson) as e: host_template.main() - LOG.info(str(e.value.host_template_output)) + assert e.value.changed == False diff --git a/tests/unit/plugins/modules/host_template_info/test_host_template_info.py b/tests/unit/plugins/modules/host_template_info/test_host_template_info.py index d71c6c83..1e1462d5 100644 --- a/tests/unit/plugins/modules/host_template_info/test_host_template_info.py +++ b/tests/unit/plugins/modules/host_template_info/test_host_template_info.py @@ -1,4 +1,6 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,63 +19,175 @@ __metaclass__ = type import logging -import os import pytest +from collections.abc import Generator +from pathlib import Path + +from cm_client import ( + ApiHostTemplate, + ApiHostTemplateList, + ApiHostRef, + ApiRole, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiService, + HostTemplatesResourceApi, +) +from cm_client.rest import ApiException + from ansible_collections.cloudera.cluster.plugins.modules import host_template_info + +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, + deregister_service, + register_service, ) LOG = logging.getLogger(__name__) -@pytest.fixture -def conn(): - conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) +@pytest.fixture(scope="module") +def zookeeper(cm_api_client, base_cluster, request) -> Generator[ApiService]: + # Keep track of the provisioned service(s) + service_registry = list[ApiService]() + + # Get the current cluster hosts + hosts = get_cluster_hosts(cm_api_client, base_cluster) + + id = Path(request.node.parent.name).stem + + zk_service = ApiService( + name=f"test-zk-{id}", + type="ZOOKEEPER", + display_name=f"ZooKeeper ({id})", + # Add a SERVER role (so we can start the service -- a ZK requirement!) + roles=[ApiRole(type="SERVER", host_ref=ApiHostRef(hosts[0].host_id))], + ) + + # Provision and yield the created service + yield register_service( + api_client=cm_api_client, + registry=service_registry, + cluster=base_cluster, + service=zk_service, + ) - if os.getenv("CM_HOST", None): - conn.update(host=os.getenv("CM_HOST")) + # Remove the created service + deregister_service(api_client=cm_api_client, registry=service_registry) - if os.getenv("CM_PORT", None): - conn.update(port=os.getenv("CM_PORT")) - if os.getenv("CM_ENDPOINT", None): - conn.update(url=os.getenv("CM_ENDPOINT")) +def test_host_template_info_missing_cluster(conn, module_args): + module_args( + { + **conn, + } + ) - if os.getenv("CM_PROXY", None): - conn.update(proxy=os.getenv("CM_PROXY")) + with pytest.raises(AnsibleFailJson, match="missing required arguments: cluster"): + host_template_info.main() - return { - **conn, - "verify_tls": "no", - "debug": "no", - } +def test_host_template_info_named( + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + host_template_factory, +): + id = f"pytest-{request.node.name}" + + base_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) -def test_all_host_templates(module_args, conn): - conn.update( - cluster="cloudera.cluster.example", + host_template_factory( + cluster=base_cluster, + host_template=ApiHostTemplate( + name=id, + role_config_group_refs=[ + ApiRoleConfigGroupRef(role_config_group_name=base_rcg.name), + ], + ), ) - module_args(conn) + module_args( + { + **conn, + "cluster": base_cluster.name, + "name": id, + } + ) with pytest.raises(AnsibleExitJson) as e: host_template_info.main() - LOG.info(str(e.value.host_templates_output)) + assert len(e.value.host_templates) == 1 + + +def test_host_template_info_not_found(conn, module_args, base_cluster): + module_args( + { + **conn, + "cluster": base_cluster.name, + "name": "not_found", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_template_info.main() + assert len(e.value.host_templates) == 0 + + +def test_host_template_info_all( + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + host_template_factory, +): + id = f"pytest-{request.node.name}" + + base_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) -def test_single_host_template(module_args, conn): - conn.update( - cluster="cloudera.cluster.example", - name="MyTemplate13", + host_template_factory( + cluster=base_cluster, + host_template=ApiHostTemplate( + name=id, + role_config_group_refs=[ + ApiRoleConfigGroupRef(role_config_group_name=base_rcg.name), + ], + ), ) - module_args(conn) + module_args( + { + **conn, + "cluster": base_cluster.name, + } + ) with pytest.raises(AnsibleExitJson) as e: host_template_info.main() - LOG.info(str(e.value.host_templates_output)) + assert len(e.value.host_templates) == 1