From e1d5de387d57fe9fc2834ab2c5ff6dacfe62b7a4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:46 -0400 Subject: [PATCH 01/30] Update get_cluster_hosts() utility to read host config details Signed-off-by: Webster Mudge --- plugins/module_utils/cluster_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/module_utils/cluster_utils.py b/plugins/module_utils/cluster_utils.py index f3f90caa..0d8ff1a9 100644 --- a/plugins/module_utils/cluster_utils.py +++ b/plugins/module_utils/cluster_utils.py @@ -25,6 +25,7 @@ ApiCluster, ApiHost, ClustersResourceApi, + HostsResourceApi, ) @@ -53,10 +54,19 @@ def parse_cluster_result(cluster: ApiCluster) -> dict: # TODO Convert to use cluster_name vs the ApiCluster object for broader usage in pytest fixtures def get_cluster_hosts(api_client: ApiClient, cluster: ApiCluster) -> list[ApiHost]: - return ( + hosts = ( ClustersResourceApi(api_client) .list_hosts( cluster_name=cluster.name, ) .items ) + + host_api = HostsResourceApi(api_client) + + for h in hosts: + h.config = host_api.read_host_config( + host_id=h.host_id, + ) + + return hosts From 7a0927c72e51d870898c6737c16ca59b432526d3 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:47 -0400 Subject: [PATCH 02/30] Add parse_host_result() and create_host_model() utility functions Signed-off-by: Webster Mudge --- plugins/module_utils/host_utils.py | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/plugins/module_utils/host_utils.py b/plugins/module_utils/host_utils.py index 293a17b5..751aaa2a 100644 --- a/plugins/module_utils/host_utils.py +++ b/plugins/module_utils/host_utils.py @@ -18,12 +18,68 @@ from cm_client import ( ApiClient, + ApiConfig, + ApiConfigList, + ApiEntityTag, ApiHost, ApiHostRef, HostsResourceApi, ) from cm_client.rest import ApiException +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + normalize_output, +) + + +HOST_OUTPUT = [ + "host_id", + "ip_address", + "hostname", + "rack_id", + "last_heartbeat", + # 'role_refs': 'list[ApiRoleRef]', + "health_summary", + "health_checks", + #'host_url': 'str', + "maintenance_mode", + "commission_state", + "maintenance_owners", + #'config': 'ApiConfigList', + "num_cores", + "num_physical_cores", + "total_phys_mem_bytes", + #'entity_status': 'ApiEntityStatus', + #'cluster_ref': 'ApiClusterRef', + "distribution", + "tags", +] + + +def parse_host_result(host: ApiHost) -> dict: + output = dict() + + # Retrieve only the cluster_name if it exists + if host.cluster_ref is not None: + output.update(cluster_name=host.cluster_ref.cluster_name) + else: + output.update(cluster_name=None) + + # Parse the host itself + output.update(normalize_output(host.to_dict(), HOST_OUTPUT)) + + # Parse the host configurations + if host.config is not None: + output.update(config={c.name: c.value for c in host.config.items}) + + # Parse the role names (only the names) + if host.role_refs is not None: + output.update( + roles=[r.role_name for r in host.role_refs], + ) + + return output + def get_host( api_client: ApiClient, hostname: str = None, host_id: str = None @@ -82,3 +138,34 @@ def get_host_ref( return ApiHostRef(host.host_id, host.hostname) else: return None + + +def create_host_model( + api_client: ApiClient, + hostname: str, + ip_address: str, # TODO Check! + rack_id: str = None, + config: dict = None, + # host_template: str = None, # TODO Check! + # roles: list[ApiRole] = None, # TODO Check! + # role_config_groups: list[ApiRoleConfigGroup] = None, # TODO Check! + tags: dict = None, +) -> ApiHost: + # Set up the hostname and IP address + host = ApiHost(hostname=hostname, ip_address=ip_address) + + # Rack ID + if rack_id: + host.rack_id = rack_id + + # Configuration + if config: + host.config = ApiConfigList( + items=[ApiConfig(name=k, value=v) for k, v in config.items()] + ) + + # Tags + if tags: + host.tags = [ApiEntityTag(k, v) for k, v in tags.items()] + + return host From 45574aa7fb09e1f7db46ed19ba99c4c8a72096c9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:47 -0400 Subject: [PATCH 03/30] Refactor host module and tests using pytest fixtures Signed-off-by: Webster Mudge --- plugins/modules/host.py | 255 +++++--- tests/unit/plugins/modules/host/test_host.py | 578 +++++++++++++++++-- 2 files changed, 684 insertions(+), 149 deletions(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index b9278de2..3b5191ac 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -12,22 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, -) -from cm_client import ApiHost, ApiHostList -from cm_client import ClustersResourceApi -from cm_client import HostsResourceApi -from cm_client.rest import ApiException - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" ---- module: host short_description: Manage hosts within Cloudera Manager description: @@ -85,7 +70,6 @@ """ EXAMPLES = r""" ---- - name: Create a host cloudera.cluster.host: host: "example.cloudera.host" @@ -121,12 +105,9 @@ password: "S&peR4Ec*re" cluster_hostname: "Ecs_node_01" state: "absent" - - """ RETURN = r""" ---- cloudera_manager: description: Details about Cloudera Manager Host type: dict @@ -209,86 +190,158 @@ returned: optional """ +from cm_client import ( + ApiHost, + ApiHostList, + ClustersResourceApi, + HostsResourceApi, +) +from cm_client.rest import ApiException + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( + create_host_model, + get_host, + parse_host_result, +) + -class ClouderaHost(ClouderaManagerModule): +class ClusterHost(ClouderaManagerModule): def __init__(self, module): - super(ClouderaHost, self).__init__(module) + super(ClusterHost, self).__init__(module) - # Initialize the return values - self.cluster_hostname = self.get_param("cluster_hostname") + # Set the parameters self.name = self.get_param("name") - self.host_ip = self.get_param("host_ip") - self.state = self.get_param("state") + self.cluster = self.get_param("cluster") + self.host_id = self.get_param("host_id") + self.ip_address = self.get_param("ip_address") self.rack_id = self.get_param("rack_id") + self.config = self.get_param("config") + self.host_template = self.get_param("host_template") + self.roles = self.get_param("roles") + self.role_config_groups = self.get_param("role_config_groups") + self.tags = self.get_param("tags") + self.purge = self.get_param("purge") + self.maintenance = self.get_param("maintenance") + self.state = self.get_param("state") + + # Initialize the return values + self.output = {} + self.diff = dict(before=dict(), after=dict()) + self.changed = False + # Execute the logic self.process() @ClouderaManagerModule.handle_process def process(self): - cluster_api_instance = ClustersResourceApi(self.api_client) - host_api_instance = HostsResourceApi(self.api_client) - self.host_output = {} - self.changed = False - existing = None + cluster_api = ClustersResourceApi(self.api_client) + host_api = HostsResourceApi(self.api_client) + + current = None try: - existing = host_api_instance.read_host( - host_id=self.cluster_hostname - ).to_dict() + current = get_host( + api_client=self.api_client, + hostname=self.name, + host_id=self.host_id, + ) except ApiException as ex: if ex.status != 404: raise ex - if self.state == "present": - if existing: - host_id = existing["host_id"] - else: - host_params = { - "hostname": self.cluster_hostname, - "ip_address": self.host_ip, - } - if self.rack_id: - host_params["rack_id"] = self.rack_id + if self.state == "absent": + if current: + self.changed = True + + if self.module._diff: + self.diff.update(before=parse_host_result(current), after=dict()) + if not self.module.check_mode: - host_list = ApiHostList(items=[ApiHost(**host_params)]) - create_host = host_api_instance.create_hosts(body=host_list) - host_id = create_host.items[0].host_id + host_api.delete_host(host_id=current["host_id"]) + + elif self.state == "present": + if current: + if self.ip_address and self.ip_address != current.ip_address: + self.module.fail_json( + msg="Invalid host configuration. To update the host IP address, please remove and then add the host." + ) + + if self.rack_id and self.rack_id != current.rack_id: self.changed = True - self.host_output = host_api_instance.read_host(host_id=host_id).to_dict() - elif self.state == "absent": - if existing: + if self.module._diff: + self.diff["before"].update(rack_id=current.rack_id) + self.diff["after"].update(rack_id=self.rack_id) + + current.rack_id = self.rack_id + + # Currently, update_host() only handles rack_id, so executing here, not further in the logic + if not self.module.check_mode: + current = host_api.update_host( + host_id=current.host_id, body=current + ) + + # Handle host template assignment + # TODO Read the RCGs for the HT, index by type, and then compare vs the actual role types + # on the instance. If any deltas (read: additions), reapply the HT. + + # Handle role config group assignment + + # Handle role override assignment + + else: + if self.ip_address is None: + self.module.fail_json( + "Invalid host configuration. IP address is required for new hosts." + ) + + current = create_host_model( + api_client=self.api_client, + hostname=self.name, + ip_address=self.ip_address, + rack_id=self.rack_id, + config=self.config, + tags=self.tags, + ) + + self.changed = True + + if self.module._diff: + self.diff.update(before=dict(), after=parse_host_result(current)) + if not self.module.check_mode: - self.host_output = host_api_instance.delete_host( - host_id=existing["host_id"] - ).to_dict() - self.changed = True + current = host_api.create_hosts( + body=ApiHostList(items=[current]) + ).items[0] elif self.state in ["attached", "detached"]: try: - cluster_api_instance.read_cluster(cluster_name=self.name).to_dict() + cluster_api.read_cluster(cluster_name=self.name).to_dict() except ApiException as ex: if ex.status == 404: self.module.fail_json(msg=f"Cluster does not exist: {self.name}") if self.state == "attached": - if existing: + if current: try: if not self.module.check_mode: host_list = ApiHostList( items=[ ApiHost( hostname=self.cluster_hostname, - host_id=existing["host_id"], + host_id=current["host_id"], ) ] ) - cluster_api_instance.add_hosts( + cluster_api.add_hosts( cluster_name=self.name, body=host_list ) - host_id = existing["host_id"] + host_id = current["host_id"] self.changed = True except ApiException as ex: if ex.status == 400: @@ -296,15 +349,13 @@ def process(self): else: host_params = { "hostname": self.cluster_hostname, - "ip_address": self.host_ip, + "ip_address": self.ip_address, } if self.rack_id: host_params["rack_id"] = self.rack_id if not self.module.check_mode: new_host_param = ApiHostList(items=[ApiHost(**host_params)]) - create_host = host_api_instance.create_hosts( - body=new_host_param - ) + create_host = host_api.create_hosts(body=new_host_param) host_list = ApiHostList( items=[ ApiHost( @@ -313,7 +364,7 @@ def process(self): ) ] ) - add_host = cluster_api_instance.add_hosts( + add_host = cluster_api.add_hosts( cluster_name=self.name, body=host_list ) host_id = add_host.items[0].host_id @@ -321,51 +372,81 @@ def process(self): elif self.state == "detached": if ( - existing - and existing.get("cluster_ref") - and existing["cluster_ref"].get("cluster_name") + current + and current.get("cluster_ref") + and current["cluster_ref"].get("cluster_name") ): if not self.module.check_mode: - cluster_api_instance.remove_host( - cluster_name=existing["cluster_ref"]["cluster_name"], - host_id=existing["host_id"], + cluster_api.remove_host( + cluster_name=current["cluster_ref"]["cluster_name"], + host_id=current["host_id"], ) - host_id = existing["host_id"] + host_id = current["host_id"] self.changed = True - self.host_output = host_api_instance.read_host( - host_id=self.cluster_hostname - ).to_dict() + # Refresh if state has changed + if self.changed: + self.output = parse_host_result(host_api.read_host(host_id=current.host_id)) + else: + self.output = parse_host_result(current) def main(): module = ClouderaManagerModule.ansible_module( argument_spec=dict( - cluster_hostname=dict(required=True, type="str"), - name=dict(required=False, type="str"), - host_ip=dict(required=False, type="str", aliases=["cluster_host_ip"]), - rack_id=dict(required=False, type="str"), + name=dict(aliases=["cluster_hostname"]), + cluster=dict(aliases=["cluster_name"]), + host_id=dict(), + ip_address=dict(aliases=["host_ip"]), + rack_id=dict(), + config=dict(type="dict", aliases=["parameters", "params"]), + host_template=dict(aliases=["template"]), + roles=dict( + type="list", + elements="dict", + options=dict( + service=dict(required=True, aliases=["service_name"]), + type=dict(required=True, aliases=["role_type"]), + config=dict(type=dict, aliases=["parameters", "params"]), + ), + ), + role_config_groups=dict( + type="list", + elements="dict", + options=dict( + service=dict(required=True, aliases=["service_name"]), + type=dict(aliases=["role_type"]), + name=dict(), + ), + required_one_of=[ + ("type", "name"), + ], + ), + tags=dict(type="dict"), + purge=dict(type="bool", default=False), + maintenance=dict(type="bool", aliases=["maintenance_mode"]), state=dict( - type="str", default="present", choices=["present", "absent", "attached", "detached"], ), ), - supports_check_mode=True, + required_one_of=[ + ("name", "host_id"), + ], required_if=[ - ("state", "attached", ("name", "host_ip"), False), - ("state", "detached", ("name",), False), - ("state", "present", ("host_ip",), False), + ("state", "attached", ("cluster",), False), + # ("state", "attached", ("name", "ip_address",), False), + # ("state", "detached", ("name",), False), + # ("state", "present", ("ip_address",), False), # TODO Move to execution check ], + supports_check_mode=True, ) - result = ClouderaHost(module) - - changed = result.changed + result = ClusterHost(module) output = dict( - changed=changed, - cloudera_manager=result.host_output, + changed=result.changed, + host=result.output, ) if result.debug: diff --git a/tests/unit/plugins/modules/host/test_host.py b/tests/unit/plugins/modules/host/test_host.py index 3796f1e8..dd3800a6 100644 --- a/tests/unit/plugins/modules/host/test_host.py +++ b/tests/unit/plugins/modules/host/test_host.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,83 +17,537 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -import os + import logging import pytest +from collections.abc import Generator +from pathlib import Path + +from cm_client import ( + ApiConfigList, + ApiHost, + ApiHostList, + ApiHostRef, + ApiHostRefList, + ApiHostTemplate, + ApiHostTemplateList, + ApiHostRef, + ApiRole, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiRoleList, + ApiService, + ClouderaManagerResourceApi, + ClustersResourceApi, + HostsResourceApi, + HostTemplatesResourceApi, + RolesResourceApi, + ServicesResourceApi, +) +from cm_client.rest import ApiException + from ansible_collections.cloudera.cluster.plugins.modules import host +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + read_roles, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + create_host_template_model, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, + deregister_service, + register_service, ) LOG = logging.getLogger(__name__) -def test_pytest_add_host_to_cloudera_manager(module_args): - module_args( - { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", - "cluster_hostname": "cloudera.host.example", - "rack_id": "/defo", - "cluster_host_ip": "10.10.1.1", - "state": "present", - } +@pytest.fixture() +def available_hosts(cm_api_client) -> list[ApiHost]: + return [ + h + for h in HostsResourceApi(cm_api_client).read_hosts().items + if h.cluster_ref is None + ] + + +@pytest.fixture() +def cluster_hosts(cm_api_client, base_cluster) -> list[ApiHost]: + return ( + ClustersResourceApi(cm_api_client) + .list_hosts(cluster_name=base_cluster.name) + .items ) - with pytest.raises(AnsibleExitJson) as e: - host.main() - - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - - -def test_pytest_attach_host_to_cluster(module_args): - module_args( - { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", - "cluster_hostname": "cloudera.host.example", - "name": "Cluster_Example", - "rack_id": "/defo", - "cluster_host_ip": "10.10.1.1", - "state": "attached", - } + +@pytest.fixture(scope="module") +def zookeeper(cm_api_client, base_cluster, request) -> Generator[ApiService]: + # Keep track of the provisioned service(s) + service_registry = list[ApiService]() + + # Get the current cluster hosts + hosts = get_cluster_hosts(cm_api_client, base_cluster) + + id = Path(request.node.parent.name).stem + + zk_service = ApiService( + name=f"test-zk-{id}", + type="ZOOKEEPER", + display_name=f"ZooKeeper ({id})", + # Add a SERVER role (so we can start the service -- a ZK requirement!) + roles=[ApiRole(type="SERVER", host_ref=ApiHostRef(hosts[0].host_id))], ) - with pytest.raises(AnsibleExitJson) as e: - host.main() - - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) - - -def test_pytest_detach_host_from_cluster(module_args): - module_args( - { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", - "cluster_hostname": "cloudera.host.example", - "name": "Cluster_Example", - "state": "detached", - } + # Provision and yield the created service + yield register_service( + api_client=cm_api_client, + registry=service_registry, + cluster=base_cluster, + service=zk_service, ) - with pytest.raises(AnsibleExitJson) as e: - host.main() + # Remove the created service + deregister_service(api_client=cm_api_client, registry=service_registry) + + +@pytest.fixture(autouse=True) +def resettable_cluster(cm_api_client, base_cluster): + host_api = HostsResourceApi(cm_api_client) + cluster_api = ClustersResourceApi(cm_api_client) + service_api = ServicesResourceApi(cm_api_client) + role_api = RolesResourceApi(cm_api_client) + + # Keep track of attached hosts and their role assignments + prior_hosts = dict[str, (ApiHost, dict[str, ApiRole])]() + + # Get all services on the cluster + prior_services = service_api.read_services( + cluster_name=base_cluster.name, + ).items + + # For each host in the cluster, get a map of each service role type's instance + for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): + prior_roles_by_service = dict[str, dict[str, ApiRole]]() + + # And for each service in the cluster + for s in prior_services: + # Retrieve any roles for the host + prior_roles_by_service[s.name] = { + r.type: r + for r in read_roles( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=s.name, + host_id=h.host_id, + ).items + } + + # Add to the map of prior hosts + prior_hosts[h.host_id] = (h, prior_roles_by_service) + + # yield to the tests + yield base_cluster + + # Each current host + for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): + # If new, remove + if h.host_id not in prior_hosts: + cluster_api.remove_host( + cluster_name=base_cluster.name, + host_id=h.host_id, + ) + # Else, update host, host config, and roles + else: + (prior_host, prior_roles_by_service) = prior_hosts.pop(h.host_id) + host_api.update_host( + host_id=h.host_id, + body=prior_host, + ) + host_api.update_host_config( + host_id=h.host_id, + body=prior_host.config, + ) + + # Get current roles for the host by service + for s in prior_services: + current_roles = read_roles( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=s.name, + host_id=h.host_id, + ).items + + # Retrieve the map of prior service roles (by type) + prior_role_types = prior_roles_by_service.get(s.name) + + for current_role in current_roles: + # If the current has a new role type, remove it + if current_role.type not in prior_role_types: + role_api.delete_role( + cluster_name=base_cluster.name, + service_name=s.name, + role_name=current_role.name, + ) + # Else, update the role and its config with the prior settings + else: + prior_role = prior_role_types.pop(current_role.type) + + if not prior_role.config: + prior_role.config = ApiConfigList() + + role_api.update_role_config( + cluster_name=base_cluster.name, + service_name=s.name, + role_name=current_role.name, + body=prior_role.config, + ) + + # If a prior role type is missing, restore + if prior_role_types: + for r in prior_role_types: + role_api.create_roles( + cluster_name=base_cluster.name, + service_name=r.service_ref.service_name, + body=ApiRoleList(items=[r]), + ) + + # If missing, restore host and roles + if prior_hosts: + cluster_api.add_hosts( + cluster_name=base_cluster.name, + body=ApiHostRefList( + items=[ + ApiHostRef(host_id=prior_host.host_id, hostname=prior_host.hostname) + for prior_host in prior_hosts + ] + ), + ) + + +class TestHostArgSpec: + def test_host_missing_required(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises( + AnsibleFailJson, match="one of the following is required: name, host_id" + ) as e: + host.main() + + def test_host_missing_attached_ip_address(self, conn, module_args): + module_args( + { + **conn, + "name": "example", + "state": "attached", + } + ) + + with pytest.raises( + AnsibleFailJson, + match="state is attached but all of the following are missing: cluster", + ) as e: + host.main() + + +# TODO Tackle the mutations first, as provisioning will require a host without CM agent... +@pytest.mark.skip() +class TestHostProvision: + def test_host_create_missing_ip_address(self, conn, module_args): + module_args( + { + **conn, + "name": "pytest-host", + } + ) + + with pytest.raises( + AnsibleFailJson, + match="Invalid host configuration. IP address is required for new hosts.", + ) as e: + host.main() + + def test_host_create_ip_address(self, conn, module_args, available_hosts): + module_args( + { + **conn, + "name": "pytest-host", + "ip_address": available_hosts[0].ip_address, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_create_rack_id(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_create_host_template(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_create_tags(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_create_maintenance_enabled(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + +class TestHostModification: + def test_host_update_ip_address(self, conn, module_args, cluster_hosts): + target_host = cluster_hosts[0] + + module_args( + { + **conn, + "name": target_host.hostname, + "ip_address": "10.0.0.1", + } + ) + + with pytest.raises(AnsibleFailJson, match="To update the host IP address") as e: + host.main() + + def test_host_update_rack_id(self, conn, module_args, cluster_hosts): + target_host = cluster_hosts[0] + + module_args( + { + **conn, + "name": target_host.hostname, + "rack_id": "/pytest1", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + assert e.value.host["rack_id"] == "/pytest1" + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + assert e.value.host["rack_id"] == "/pytest1" + + def test_host_update_host_template( + self, + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + cluster_hosts, + role_config_group_factory, + host_template_factory, + ): + target_host = cluster_hosts[0] + target_name = f"pytest-{Path(request.node.name)}" + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + host_template = host_template_factory( + cluster=base_cluster, + host_template=create_host_template_model( + cluster_name=base_cluster.name, + name=target_name, + role_config_groups=[target_rcg], + ), + ) + + module_args( + { + **conn, + "name": target_host.hostname, + "host_template": host_template.name, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + def test_host_update_host_template_new_role(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_update_tags(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_update_maintenance_enabled(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + def test_host_update_maintenance_disabled(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + +class TestHostAttached: + def test_host_create_invalid_cluster(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + +class TestHostDetached: + def test_host_create_invalid_cluster(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + +# def test_pytest_add_host_to_cloudera_manager(module_args): +# module_args( +# { +# "username": os.getenv("CM_USERNAME"), +# "password": os.getenv("CM_PASSWORD"), +# "host": os.getenv("CM_HOST"), +# "port": "7180", +# "verify_tls": "no", +# "debug": "no", +# "cluster_hostname": "cloudera.host.example", +# "rack_id": "/defo", +# "cluster_host_ip": "10.10.1.1", +# "state": "present", +# } +# ) + +# with pytest.raises(AnsibleExitJson) as e: +# host.main() + +# # LOG.info(str(e.value)) +# LOG.info(str(e.value.cloudera_manager)) + + +# def test_pytest_attach_host_to_cluster(module_args): +# module_args( +# { +# "username": os.getenv("CM_USERNAME"), +# "password": os.getenv("CM_PASSWORD"), +# "host": os.getenv("CM_HOST"), +# "port": "7180", +# "verify_tls": "no", +# "debug": "no", +# "cluster_hostname": "cloudera.host.example", +# "name": "Cluster_Example", +# "rack_id": "/defo", +# "cluster_host_ip": "10.10.1.1", +# "state": "attached", +# } +# ) + +# with pytest.raises(AnsibleExitJson) as e: +# host.main() + +# # LOG.info(str(e.value)) +# LOG.info(str(e.value.cloudera_manager)) + + +# def test_pytest_detach_host_from_cluster(module_args): +# module_args( +# { +# "username": os.getenv("CM_USERNAME"), +# "password": os.getenv("CM_PASSWORD"), +# "host": os.getenv("CM_HOST"), +# "port": "7180", +# "verify_tls": "no", +# "debug": "no", +# "cluster_hostname": "cloudera.host.example", +# "name": "Cluster_Example", +# "state": "detached", +# } +# ) + +# with pytest.raises(AnsibleExitJson) as e: +# host.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) +# # LOG.info(str(e.value)) +# LOG.info(str(e.value.cloudera_manager)) From 1ad0ae2f32d23bdfbf9f5166d2774220df7ea486 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:48 -0400 Subject: [PATCH 04/30] Add HostTemplateException Signed-off-by: Webster Mudge --- plugins/module_utils/host_template_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/module_utils/host_template_utils.py b/plugins/module_utils/host_template_utils.py index 0a755a71..f6ac9a27 100644 --- a/plugins/module_utils/host_template_utils.py +++ b/plugins/module_utils/host_template_utils.py @@ -24,6 +24,10 @@ ) +class HostTemplateException(Exception): + pass + + def parse_host_template(host_template: ApiHostTemplate) -> dict: return dict( name=host_template.name, From 9b21ceaffa0e66d181920c19bafc91b0a6b233f0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:49 -0400 Subject: [PATCH 05/30] Add view parameter to get_host() utility function Add get_host_roles() utility function Add reconcile_host_role_configs() utility function Add reconcile_host_role_config_groups() utility function Add reconcile_host_template_assignments() utility function Add toggle_host_role_states() utility function Add toggle_host_maintenance() utilty function Add detach_host() utility function Signed-off-by: Webster Mudge --- plugins/module_utils/host_utils.py | 643 ++++++++++++++++++++++++++++- 1 file changed, 634 insertions(+), 9 deletions(-) diff --git a/plugins/module_utils/host_utils.py b/plugins/module_utils/host_utils.py index 751aaa2a..26ce2836 100644 --- a/plugins/module_utils/host_utils.py +++ b/plugins/module_utils/host_utils.py @@ -18,18 +18,54 @@ from cm_client import ( ApiClient, + ApiCluster, ApiConfig, ApiConfigList, ApiEntityTag, ApiHost, ApiHostRef, + ApiHostRefList, + ApiHostTemplate, + ApiRole, + ApiRoleConfigGroup, + ApiRoleNameList, + ApiRoleState, + ClustersResourceApi, HostsResourceApi, + HostTemplatesResourceApi, + RolesResourceApi, + RoleConfigGroupsResourceApi, + RoleCommandsResourceApi, + ServicesResourceApi, ) from cm_client.rest import ApiException from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( normalize_output, + wait_command, + wait_bulk_commands, + ConfigListUpdates, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + HostTemplateException, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + create_role, + provision_service_role, + read_role, + read_roles, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) + + +class HostException(Exception): + pass + + +class HostMaintenanceStateException(Exception): + pass HOST_OUTPUT = [ @@ -71,18 +107,25 @@ def parse_host_result(host: ApiHost) -> dict: # Parse the host configurations if host.config is not None: output.update(config={c.name: c.value for c in host.config.items}) + else: + output.update(config=None) # Parse the role names (only the names) if host.role_refs is not None: output.update( roles=[r.role_name for r in host.role_refs], ) + else: + output.update(roles=None) return output def get_host( - api_client: ApiClient, hostname: str = None, host_id: str = None + api_client: ApiClient, + hostname: str = None, + host_id: str = None, + view: str = "summary", ) -> ApiHost: """Retrieve a Host by either hostname or host ID. @@ -97,23 +140,25 @@ def get_host( Returns: ApiHost: Host object. If not found, returns None. """ + host_api = HostsResourceApi(api_client) if hostname: - return next( - ( - h - for h in HostsResourceApi(api_client).read_hosts().items - if h.hostname == hostname - ), + host = next( + (h for h in host_api.read_hosts(view=view).items if h.hostname == hostname), None, ) else: try: - return HostsResourceApi(api_client).read_host(host_id) + host = host_api.read_host(host_id=host_id, view=view) except ApiException as ex: if ex.status != 404: raise ex else: - return None + host = None + + if host is not None: + host.config = host_api.read_host_config(host.host_id) + + return host def get_host_ref( @@ -140,6 +185,21 @@ def get_host_ref( return None +def get_host_roles( + api_client: ApiClient, + host: ApiHost, +) -> list[ApiRole]: + return [ + read_role( + api_client=api_client, + cluster_name=role_ref.cluster_name, + service_name=role_ref.service_name, + role_name=role_ref.role_name, + ) + for role_ref in host.role_refs + ] + + def create_host_model( api_client: ApiClient, hostname: str, @@ -169,3 +229,568 @@ def create_host_model( host.tags = [ApiEntityTag(k, v) for k, v in tags.items()] return host + + +# Only updates, no assignment +def reconcile_host_role_configs( + api_client: ApiClient, + host: ApiHost, + role_configs: list[dict], # service, type, and config (optional) + purge: bool, + check_mode: bool, + message: str = None, +) -> tuple[list[dict], list[dict]]: + + diff_before, diff_after = list[dict](), list[dict]() + + role_api = RolesResourceApi(api_client) + + for incoming_role_config in role_configs: + # Retrieve the current role by service and type + current_role = next( + iter( + read_roles( + api_client=api_client, + cluster_name=host.cluster_ref.cluster_name, + service_name=incoming_role_config["service"], + type=incoming_role_config["type"], + host_id=host.host_id, + ).items + ), + None, + ) + + # If no existing role of service and type exists, raise an error + if current_role is None: + raise HostException( + f"No role of type, '{incoming_role_config['type']}', found for service, '{incoming_role_config['service']}', on cluster, '{host.cluster_ref.cluster_name}'" + ) + + # Reconcile role override configurations + if incoming_role_config["config"] or purge: + incoming_config = incoming_role_config.get("config", dict()) + + updates = ConfigListUpdates(current_role.config, incoming_config, purge) + + if updates.changed: + diff_before.append( + dict(name=current_role.name, config=current_role.config) + ) + diff_after.append(dict(name=current_role.name, config=updates.config)) + + current_role.config = updates.config + + if not check_mode: + role_api.update_role_config( + cluster_name=host.cluster_ref.cluster_name, + service_name=current_role.service_ref.service_name, + role_name=current_role.name, + body=current_role.config, + message=message, + ) + + return (diff_before, diff_after) + + +def reconcile_host_role_config_groups( + api_client: ApiClient, + cluster: ApiCluster, + host: ApiHost, + role_config_groups: list[dict], # service, type (optional), name (optional) + purge: bool, + check_mode: bool, +) -> tuple[list[dict], list[dict]]: + + rcg_api = RoleConfigGroupsResourceApi(api_client) + + # Index by all cluster role config groups by name + cluster_rcgs = _read_cluster_rcgs( + api_client=api_client, + cluster=cluster, + ) + + # Index the declared role config groups by name + declared_rcgs = dict[str, ApiRoleConfigGroup]() # RCG name -> RCG + for rcg in role_config_groups: + # If the role config group is a base, retrieve its name + if rcg.get("name", None) is None: + base_rcg = get_base_role_config_group( + api_client=api_client, + cluster_name=host.cluster_ref.cluster_name, + service_name=rcg["service"], + role_type=rcg["type"], + ) + if base_rcg is None: + raise HostException( + f"Base role config group for type, '{rcg['type']}', not found." + ) + declared_rcgs[base_rcg.name] = base_rcg + # Else, confirm the custom role config group and use its name + else: + custom_rcg = rcg_api.read_role_config_group( + cluster_name=host.cluster_ref.cluster_name, + service_name=rcg["service"], + role_config_group_name=rcg["name"], + ) + if custom_rcg is None: + raise HostException( + f"Named role config group, '{rcg['name']}', not found." + ) + declared_rcgs[custom_rcg.name] = custom_rcg + + # Retrieve the associated role config groups from each installed role on the host + host_rcgs = _read_host_rcgs( + api_client=api_client, + host=host, + cluster_rcgs=cluster_rcgs, + ) + + # Reconcile the role config groups on the host with the declared role config groups + return _reconcile_host_rcgs( + api_client=api_client, + host=host, + cluster_rcgs=cluster_rcgs, + declared_rcgs=declared_rcgs, + host_rcgs=host_rcgs, + purge=purge, + check_mode=check_mode, + ) + + +def _read_cluster_rcgs( + api_client: ApiClient, + cluster: ApiCluster, +) -> dict[str, ApiRoleConfigGroup]: + service_api = ServicesResourceApi(api_client) + rcg_api = RoleConfigGroupsResourceApi(api_client) + return { + rcg.name: rcg + for service in service_api.read_services(cluster_name=cluster.name).items + for rcg in rcg_api.read_role_config_groups( + cluster_name=cluster.name, service_name=service.name + ).items + } + + +def _read_host_rcgs( + api_client: ApiClient, + host: ApiHost, + cluster_rcgs: dict[str, ApiRoleConfigGroup], +) -> dict[str, ApiRoleConfigGroup]: + current_rcgs = dict[str, ApiRoleConfigGroup]() + for role_ref in host.role_refs: + role = read_role( + api_client=api_client, + cluster_name=role_ref.cluster_name, + service_name=role_ref.service_name, + role_name=role_ref.role_name, + ) + if role.role_config_group_ref.role_config_group_name in cluster_rcgs: + current_rcgs[ + role.role_config_group_ref.role_config_group_name + ] = cluster_rcgs[role.role_config_group_ref.role_config_group_name] + else: + raise Exception( + f"Invalid role config group reference, '{role.role_config_group_ref.role_config_group_name}', on host, {host.hostname}" + ) + return current_rcgs + + +def _reconcile_host_rcgs( + api_client: ApiClient, + host: ApiHost, + cluster_rcgs: dict[str, ApiRoleConfigGroup], + declared_rcgs: dict[str, ApiRoleConfigGroup], + host_rcgs: dict[str, ApiRoleConfigGroup], + purge: bool, + check_mode: bool, +) -> tuple[list[dict], list[dict]]: + + diff_before, diff_after = list[dict](), list[dict]() + + role_api = RolesResourceApi(api_client) + + additions = set(declared_rcgs.keys()) - set(host_rcgs.keys()) + deletions = set(host_rcgs.keys()) - set(declared_rcgs.keys()) + + # If the host template has additional assignments + if additions: + for add_rcg_name in additions: + + # Retrieve the role config group by name from the cluster + add_rcg = cluster_rcgs[add_rcg_name] + + # Create the role instance model using the role config group + created_role = create_role( + api_client=api_client, + host_id=host.host_id, + cluster_name=add_rcg.service_ref.cluster_name, + service_name=add_rcg.service_ref.service_name, + role_type=add_rcg.role_type, + role_config_group=add_rcg.name, + ) + + diff_before.append(dict()) + diff_after.append(created_role.to_dict()) + + if not check_mode: + provision_service_role( + api_client=api_client, + cluster_name=add_rcg.service_ref.cluster_name, + service_name=add_rcg.service_ref.service_name, + role=created_role, + ) + + # If the host has extraneous assignments + if deletions and purge: + for del_rcg_name in deletions: + + # Retrieve the current role config group by name + del_rcg = cluster_rcgs[del_rcg_name] # current_rcgs[del_rcg_name] + + # Retrieve the role instance on the host via the role config group's type + del_roles = read_roles( + api_client=api_client, + host_id=host.host_id, + cluster_name=del_rcg.service_ref.cluster_name, + service_name=del_rcg.service_ref.service_name, + type=del_rcg.role_type, + ).items + + if not del_roles: + raise Exception( + f"Error reading role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" + ) + if len(del_roles) != 1: + raise Exception( + f"Error, multiple instances for role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" + ) + + diff_before.append(del_roles[0].to_dict()) + diff_after.append(dict()) + + if not check_mode: + role_api.delete_role( + cluster_name=del_roles[0].service_ref.cluster_name, + service_name=del_roles[0].service_ref.service_name, + role_name=del_roles[0].name, + ) + return (diff_before, diff_after) + + +def reconcile_host_template_assignments( + api_client: ApiClient, + cluster: ApiCluster, + host: ApiHost, + host_template: ApiHostTemplate, + purge: bool, + check_mode: bool, +) -> tuple[list[dict], list[dict]]: + + # diff_before, diff_after = list[dict](), list[dict]() + + host_template_api = HostTemplatesResourceApi(api_client) + # service_api = ServicesResourceApi(api_client) + # rcg_api = RoleConfigGroupsResourceApi(api_client) + # role_api = RolesResourceApi(api_client) + + # Index by all cluster role config groups by name + # cluster_rcg_map = { + # rcg.name: rcg + # for service in service_api.read_services(cluster_name=cluster.name).items + # for rcg in rcg_api.read_role_config_groups( + # cluster_name=cluster.name, service_name=service.name + # ).items + # } + cluster_rcg_map = _read_cluster_rcgs( + api_client=api_client, + cluster=cluster, + ) + + # Index the host template role config groups by name + ht_rcgs = dict[str, ApiRoleConfigGroup]() # RCG name -> RCG + for rcg_ref in host_template.role_config_group_refs: + if rcg_ref.role_config_group_name in cluster_rcg_map: + ht_rcgs[rcg_ref.role_config_group_name] = cluster_rcg_map[ + rcg_ref.role_config_group_name + ] + else: + raise HostTemplateException( + f"Invalid role config group reference, '{rcg_ref.role_config_group_name}', in host template, {host_template.name}" + ) + + # Retrieve the associated role config groups from each installed role + # current_rcgs = dict[str, ApiRoleConfigGroup]() + # for role_ref in host.role_refs: + # role = read_role( + # api_client=api_client, + # cluster_name=role_ref.cluster_name, + # service_name=role_ref.service_name, + # role_name=role_ref.role_name, + # ) + # if role.role_config_group_ref.role_config_group_name in cluster_rcg_map: + # current_rcgs[ + # role.role_config_group_ref.role_config_group_name + # ] = cluster_rcg_map[role.role_config_group_ref.role_config_group_name] + # else: + # raise Exception( + # f"Invalid role config group reference, '{role.role_config_group_ref.role_config_group_name}', on host, {host.hostname}" + # ) + current_rcgs = _read_host_rcgs( + api_client=api_client, + host=host, + cluster_rcgs=cluster_rcg_map, + ) + + # If the host has no current role assignments + if not current_rcgs: + diff_before, diff_after = list[dict](), list[dict]() + + for add_rcg in ht_rcgs.values(): + diff_before.append(dict()) + diff_after.append( + create_role( + api_client=api_client, + host_id=host.host_id, + cluster_name=add_rcg.service_ref.cluster_name, + service_name=add_rcg.service_ref.service_name, + role_type=add_rcg.role_type, + role_config_group=add_rcg.name, + ).to_dict() + ) + + if not check_mode: + # Apply the host template + apply_cmd = host_template_api.apply_host_template( + cluster_name=cluster.name, + host_template_name=host_template.name, + start_roles=False, + body=ApiHostRefList( + items=[ApiHostRef(host_id=host.host_id, hostname=host.hostname)] + ), + ) + wait_command( + api_client=api_client, + command=apply_cmd, + ) + + return (diff_before, diff_after) + + # Else the host has role assignments + else: + return _reconcile_host_rcgs( + api_client=api_client, + host=host, + cluster_rcgs=cluster_rcg_map, + declared_rcgs=ht_rcgs, + host_rcgs=current_rcgs, + purge=purge, + check_mode=check_mode, + ) + # additions = set(ht_rcgs.keys()) - set(current_rcgs.keys()) + # deletions = set(current_rcgs.keys()) - set(ht_rcgs.keys()) + + # # If the host template has additional assignments + # if additions: + # for add_rcg_name in additions: + + # # Retrieve the role config group by name from the cluster + # add_rcg = cluster_rcg_map[add_rcg_name] + + # # Create the role instance model using the role config group + # created_role = create_role( + # api_client=api_client, + # host_id=host.host_id, + # cluster_name=add_rcg.service_ref.cluster_name, + # service_name=add_rcg.service_ref.service_name, + # role_type=add_rcg.role_type, + # role_config_group=add_rcg.name, + # ) + + # diff_before.append(dict()) + # diff_after.append(created_role.to_dict()) + + # if not check_mode: + # provision_service_role( + # api_client=api_client, + # cluster_name=add_rcg.service_ref.cluster_name, + # service_name=add_rcg.service_ref.service_name, + # role=created_role, + # ) + + # # If the host has extraneous assignments + # if deletions and purge: + # for del_rcg_name in deletions: + + # # Retrieve the current role config group by name + # del_rcg = cluster_rcg_map[del_rcg_name] # current_rcgs[del_rcg_name] + + # # Retrieve the role instance on the host via the role config group's type + # del_roles = read_roles( + # api_client=api_client, + # host_id=host.host_id, + # cluster_name=del_rcg.service_ref.cluster_name, + # service_name=del_rcg.service_ref.service_name, + # type=del_rcg.role_type, + # ).items + + # if not del_roles: + # raise Exception( + # f"Error reading role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" + # ) + # if len(del_roles) != 1: + # raise Exception( + # f"Error, multiple instances for role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" + # ) + + # diff_before.append(del_roles[0].to_dict()) + # diff_after.append(dict()) + + # if not check_mode: + # role_api.delete_role( + # cluster_name=del_roles[0].service_ref.cluster_name, + # service_name=del_roles[0].service_ref.service_name, + # role_name=del_roles[0].name, + # ) + + # return (diff_before, diff_after) + + +def toggle_host_role_states( + api_client: ApiClient, host: ApiHost, state: str, check_mode: bool +) -> tuple[list[dict], list[dict]]: + + service_api = ServicesResourceApi(api_client) + role_api = RoleCommandsResourceApi(api_client) + + before_roles = list[dict] + after_roles = list[dict] + + service_map = dict[str, list[ApiRole]]() + + # Index each role instance on the host by its service + for role in get_host_roles(api_client, host): + if role.service_ref.service_name in service_map: + service_map[role.service_ref.service_name].append(role) + else: + service_map[role.service_ref.service_name] = [role] + + # For each service, handle the role state + for service_name, roles in service_map.items(): + service = service_api.read_service( + cluster_name=host.cluster_ref.cluster_name, + service_name=service_name, + ) + + changed_roles = list() + + for role in roles: + if state == "started" and role.role_state not in [ApiRoleState.STARTED]: + before_roles.append(dict(name=role.name, role_state=role.role_state)) + after_roles.append( + dict(name=role.name, role_state=ApiRoleState.STARTED) + ) + changed_roles.append(role) + cmd = role_api.start_command + elif state == "stopped" and role.role_state not in [ + ApiRoleState.STOPPED, + ApiRoleState.NA, + ]: + before_roles.append(dict(name=role.name, role_state=role.role_state)) + after_roles.append( + dict(name=role.name, role_state=ApiRoleState.STOPPED) + ) + changed_roles.append(role) + cmd = role_api.stop_command + elif state == "restarted": + before_roles.append(dict(name=role.name, role_state=role.role_state)) + after_roles.append( + dict(name=role.name, role_state=ApiRoleState.STARTED) + ) + changed_roles.append(role) + cmd = role_api.restart_command + + if not check_mode and changed_roles: + exec_cmds = cmd( + cluster_name=service.cluster_ref.cluster_name, + service_name=service.name, + body=ApiRoleNameList(items=changed_roles), + ) + wait_bulk_commands(api_client=api_client, commands=exec_cmds) + + return (before_roles, after_roles) + + +def toggle_host_maintenance( + api_client: ApiClient, + host: ApiHost, + maintenance: bool, + check_mode: bool, +) -> bool: + host_api = HostsResourceApi(api_client) + changed = False + + if maintenance and not host.maintenance_mode: + changed = True + cmd = host_api.enter_maintenance_mode + elif not maintenance and host.maintenance_mode: + changed = True + cmd = host_api.exit_maintenance_mode + + if not check_mode and changed: + maintenance_cmd = cmd( + host_id=host.host_id, + ) + + if maintenance_cmd.success is False: + raise HostMaintenanceStateException( + f"Unable to set Maintenance mode to '{maintenance}': {maintenance_cmd.result_message}" + ) + + return changed + + +def detach_host( + api_client: ApiClient, + host: ApiHost, + purge: bool, + check_mode: bool, +) -> tuple[list[dict], list[dict]]: + + cluster_api = ClustersResourceApi(api_client) + role_api = RolesResourceApi(api_client) + + before_role = list[dict]() + after_role = list[dict]() + + # Get all role instances on the host + current_roles = get_host_roles( + api_client=api_client, + host=host, + ) + + if current_roles and not purge: + raise HostException( + f"Unable to detach from cluster, '{host.cluster_ref.cluster_name}', due to existing role instances." + ) + + # Decommission the entirety of the host's roles + for del_role in current_roles: + before_role.append(del_role.to_dict()) + after_role.append(dict()) + + if not check_mode: + role_api.delete_role( + cluster_name=del_role.service_ref.cluster_name, + service_name=del_role.service_ref.service_name, + role_name=del_role.name, + ) + + # Detach from cluster + if not check_mode: + cluster_api.remove_host( + cluster_name=host.cluster_ref.cluster_name, + host_id=host.host_id, + ) + + return (before_role, after_role) From 9ce98fdc2607358af598d5455606294247b7a88c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:49 -0400 Subject: [PATCH 06/30] Update create_role() to handle host assignment lookup Signed-off-by: Webster Mudge --- plugins/module_utils/role_utils.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/plugins/module_utils/role_utils.py b/plugins/module_utils/role_utils.py index 37bd9923..eeb6b3d6 100644 --- a/plugins/module_utils/role_utils.py +++ b/plugins/module_utils/role_utils.py @@ -21,20 +21,19 @@ wait_commands, wait_bulk_commands, ) -from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( - get_host_ref, -) from cm_client import ( ApiClient, ApiConfig, ApiConfigList, ApiEntityTag, + ApiHostRef, ApiRole, ApiRoleList, ApiRoleConfigGroupRef, ApiRoleNameList, ApiRoleState, + HostsResourceApi, ServicesResourceApi, RoleCommandsResourceApi, RoleConfigGroupsResourceApi, @@ -42,6 +41,7 @@ MgmtRolesResourceApi, MgmtServiceResourceApi, ) +from cm_client.rest import ApiException class RoleException(Exception): @@ -261,13 +261,32 @@ def create_role( role = ApiRole(type=str(role_type).upper()) # Host assignment - host_ref = get_host_ref(api_client, hostname, host_id) + if hostname: + host_ref = next( + ( + h + for h in HostsResourceApi(api_client).read_hosts().items + if h.hostname == hostname + ), + None, + ) + elif host_id: + try: + host_ref = HostsResourceApi(api_client).read_host(host_id) + except ApiException as ex: + if ex.status != 404: + raise ex + else: + host_ref = None + else: + raise RoleException("Specify either 'hostname' or 'host_id'") + if host_ref is None: raise RoleHostNotFoundException( f"Host not found: hostname='{hostname}', host_id='{host_id}'" ) else: - role.host_ref = host_ref + role.host_ref = ApiHostRef(host_id=host_ref.host_id, hostname=host_ref.hostname) # Role config group if role_config_group: From 9b0f0a66a965339eede2bd253eafbc42153f33a0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:50 -0400 Subject: [PATCH 07/30] Update error message on missing 'type' parameter Signed-off-by: Webster Mudge --- plugins/modules/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/service.py b/plugins/modules/service.py index de18b7cb..223e8ab9 100644 --- a/plugins/modules/service.py +++ b/plugins/modules/service.py @@ -739,7 +739,7 @@ def process(self): self.changed = True if self.type is None: - self.module.fail_json(msg=f"missing required arguments: type") + self.module.fail_json(msg="missing required arguments: type") # Create and provision the service service = create_service_model( From 583977e9c4d41bce8373e0bd46315f7e0bb22a7b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:51 -0400 Subject: [PATCH 08/30] Refactor host module for updated logic and shared functions Signed-off-by: Webster Mudge --- plugins/modules/host.py | 462 +++++++++++++++++++++++++++++++--------- 1 file changed, 366 insertions(+), 96 deletions(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index 3b5191ac..665ba332 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -1,4 +1,7 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -193,22 +196,44 @@ from cm_client import ( ApiHost, ApiHostList, + ApiHostRef, + ApiHostRefList, + ApiRoleConfigGroup, ClustersResourceApi, HostsResourceApi, + HostTemplatesResourceApi, + RoleConfigGroupsResourceApi, + RolesResourceApi, ) from cm_client.rest import ApiException +from ansible.module_utils.common.text.converters import to_native + from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, + ClouderaManagerMutableModule, + ConfigListUpdates, + TagUpdates, ) from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( create_host_model, + detach_host, get_host, + get_host_roles, parse_host_result, + reconcile_host_role_configs, + reconcile_host_role_config_groups, + reconcile_host_template_assignments, + toggle_host_maintenance, + toggle_host_role_states, + HostMaintenanceStateException, + HostException, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + parse_role_result, ) -class ClusterHost(ClouderaManagerModule): +class ClusterHost(ClouderaManagerMutableModule): def __init__(self, module): super(ClusterHost, self).__init__(module) @@ -235,11 +260,14 @@ def __init__(self, module): # Execute the logic self.process() - @ClouderaManagerModule.handle_process + @ClouderaManagerMutableModule.handle_process def process(self): cluster_api = ClustersResourceApi(self.api_client) host_api = HostsResourceApi(self.api_client) + host_template_api = HostTemplatesResourceApi(self.api_client) + rcg_api = RoleConfigGroupsResourceApi(self.api_client) + role_api = RolesResourceApi(self.api_client) current = None @@ -248,11 +276,13 @@ def process(self): api_client=self.api_client, hostname=self.name, host_id=self.host_id, + view="full", ) except ApiException as ex: if ex.status != 404: raise ex + # If removing if self.state == "absent": if current: self.changed = True @@ -263,13 +293,59 @@ def process(self): if not self.module.check_mode: host_api.delete_host(host_id=current["host_id"]) - elif self.state == "present": - if current: + # Else if a known run state + elif self.state in [ + "present", + "started", + "stopped", + "restarted", + ]: + # If the host does not yet exist, so create and provision the core configuration + if not current: + self.changed = True + + if self.ip_address is None: + self.module.fail_json(msg="missing required arguments: ip_address") + + # Create and provision the host + host = create_host_model( + api_client=self.api_client, + hostname=self.name, + ip_address=self.ip_address, + rack_id=self.rack_id, + config=self.config, + tags=self.tags, + ) + + if self.module._diff: + self.diff.update(before=dict(), after=parse_host_result(host)) + + if not self.module.check_mode: + current = host_api.create_hosts( + body=ApiHostList(items=[host]) + ).items[0] + + if not current: + self.module.fail_json( + msg="Unable to create new host", + host=to_native(host.to_dict()), + ) + + # Set maintenence mode + self.handle_maintenance(current) + + # Else the host exists, so update the core configuration + else: + # Handle maintenence mode + self.handle_maintenance(current) + + # Handle IP address configuration if self.ip_address and self.ip_address != current.ip_address: self.module.fail_json( msg="Invalid host configuration. To update the host IP address, please remove and then add the host." ) + # Handle rack ID if self.rack_id and self.rack_id != current.rack_id: self.changed = True @@ -285,114 +361,292 @@ def process(self): host_id=current.host_id, body=current ) - # Handle host template assignment - # TODO Read the RCGs for the HT, index by type, and then compare vs the actual role types - # on the instance. If any deltas (read: additions), reapply the HT. - - # Handle role config group assignment + # Handle host configs + if self.config or self.purge: + if self.config is None: + self.config = dict() - # Handle role override assignment - - else: - if self.ip_address is None: - self.module.fail_json( - "Invalid host configuration. IP address is required for new hosts." + config_updates = ConfigListUpdates( + current.config, self.config, self.purge ) - current = create_host_model( - api_client=self.api_client, - hostname=self.name, - ip_address=self.ip_address, - rack_id=self.rack_id, - config=self.config, - tags=self.tags, - ) + if config_updates.changed: + self.changed = True - self.changed = True + if self.module._diff: + self.diff["before"].update( + config=config_updates.diff["before"] + ) + self.diff["after"].update( + config=config_updates.diff["after"] + ) - if self.module._diff: - self.diff.update(before=dict(), after=parse_host_result(current)) + if not self.module.check_mode: + host_api.update_host_config( + host_id=current.host_id, + message=self.message, + body=config_updates.config, + ) - if not self.module.check_mode: - current = host_api.create_hosts( - body=ApiHostList(items=[current]) - ).items[0] + # Handle tags + if self.tags or self.purge: + if self.tags is None: + self.tags = dict() - elif self.state in ["attached", "detached"]: + tag_updates = TagUpdates(current.tags, self.tags, self.purge) - try: - cluster_api.read_cluster(cluster_name=self.name).to_dict() - except ApiException as ex: - if ex.status == 404: - self.module.fail_json(msg=f"Cluster does not exist: {self.name}") + if tag_updates.changed: + self.changed = True + + if self.module._diff: + self.diff["before"].update(tags=tag_updates.diff["before"]) + self.diff["after"].update(tags=tag_updates.diff["after"]) - if self.state == "attached": - if current: - try: if not self.module.check_mode: - host_list = ApiHostList( - items=[ - ApiHost( - hostname=self.cluster_hostname, - host_id=current["host_id"], + if tag_updates.deletions: + host_api.delete_tags( + hostname=current.hostname, + body=tag_updates.deletions, + ) + + if tag_updates.additions: + host_api.add_tags( + hostname=current.hostname, + body=tag_updates.additions, + ) + + # Handle attaching and detaching from clusters + if self.cluster or self.purge: + + # If detaching from a cluster, address the role decommissioning + if self.cluster is None and self.purge: + + # Only remove the roles with a cluster reference + if current.cluster_ref is not None: + self.changed = True + + current_roles = get_host_roles(self.api_client, host=current) + + if self.module._diff: + self.diff["before"].update( + cluster=current.cluster_ref.cluster_name, roles=[] + ) + self.diff["after"].update(cluster="", roles=[]) + + for role in current_roles: + if role.service_ref.cluster_name is not None: + if self.module._diff: + self.diff["before"]["roles"].append( + parse_role_result(role) + ) + + if not self.module.check_mode: + role_api.delete_role( + cluster_name=role.service_ref.cluster_name, + service_name=role.service_ref.service_name, + role_name=role.name, ) - ] + + if not self.module.check_mode: + cluster_api.remove_host( + cluster_name=current.cluster_ref.cluster_name, + host_id=current.host_id, + ) + # Else if cluster is defined + elif self.cluster: + try: + cluster = cluster_api.read_cluster(cluster_name=self.cluster) + except ApiException as ex: + if ex.status == 404: + self.module.fail_json( + msg=f"Cluster does not exist: {self.cluster}" ) + + # Handle new cluster membership + if current.cluster_ref is None: + self.changed = True + + if self.module._diff: + self.diff["before"].update(cluster="") + self.diff["after"].update(cluster=cluster.name) + + if not self.module.check_mode: cluster_api.add_hosts( - cluster_name=self.name, body=host_list + cluster_name=cluster.name, + body=ApiHostRefList( + items=[ + ApiHostRef( + host_id=current.host_id, + hostname=current.hostname, + ) + ] + ), ) - host_id = current["host_id"] - self.changed = True - except ApiException as ex: - if ex.status == 400: - pass - else: - host_params = { - "hostname": self.cluster_hostname, - "ip_address": self.ip_address, - } - if self.rack_id: - host_params["rack_id"] = self.rack_id - if not self.module.check_mode: - new_host_param = ApiHostList(items=[ApiHost(**host_params)]) - create_host = host_api.create_hosts(body=new_host_param) - host_list = ApiHostList( - items=[ - ApiHost( - hostname=self.cluster_hostname, - host_id=create_host.items[0].host_id, - ) - ] - ) - add_host = cluster_api.add_hosts( - cluster_name=self.name, body=host_list - ) - host_id = add_host.items[0].host_id + + # Handle cluster migration + elif current.cluster_ref.cluster_name != cluster.name: self.changed = True - elif self.state == "detached": - if ( - current - and current.get("cluster_ref") - and current["cluster_ref"].get("cluster_name") - ): - if not self.module.check_mode: - cluster_api.remove_host( - cluster_name=current["cluster_ref"]["cluster_name"], - host_id=current["host_id"], + # Detach from cluster + (before_detach, after_detach) = detach_host( + api_client=self.api_client, + host=current, + purge=self.purge, + check_mode=self.module.check_mode, ) - host_id = current["host_id"] - self.changed = True - # Refresh if state has changed - if self.changed: - self.output = parse_host_result(host_api.read_host(host_id=current.host_id)) + # Attach to new cluster + if not self.module.check_mode: + cluster_api.add_hosts( + cluster_name=cluster.name, + body=ApiHostRefList( + items=[ + ApiHostRef( + host_id=current.host_id, + hostname=current.hostname, + ) + ] + ), + ) + + if self.module._diff: + self.diff["before"].update( + cluster=current.cluster_ref.cluster_name, + roles=before_detach, + ) + self.diff["after"].update( + cluster=cluster.name, + roles=after_detach, + ) + + # Handle host template assignments (argspec enforces inclusion of cluster) + if self.host_template: + try: + ht = host_template_api.read_host_template( + cluster_name=cluster.name, + host_template_name=self.host_template, + ) + except ApiException as ex: + if ex.status == 404: + self.module.fail_json( + msg=f"Host template, '{self.host_template}', does not exist on cluster, '{cluster.name}'" + ) + + (before_ht, after_ht) = reconcile_host_template_assignments( + api_client=self.api_client, + cluster=cluster, + host=current, + host_template=ht, + purge=self.purge, + check_mode=self.module.check_mode, + ) + + if before_ht or after_ht: + self.changed = True + if self.module._diff: + self.diff["before"].update(roles=before_ht) + self.diff["after"].update(roles=after_ht) + + # Handle role config group assignment (argspec enforces inclusion of cluster) + # if self.role_config_groups or (not self.host_template and self.purge): + if self.role_config_groups: + # if self.role_config_groups is None: + # self.role_config_groups = list() + + try: + (before_rcg, after_rcg) = reconcile_host_role_config_groups( + api_client=self.api_client, + cluster=cluster, + host=current, + role_config_groups=self.role_config_groups, + purge=self.purge, + check_mode=self.module.check_mode, + ) + except HostException as he: + self.module.fail_json(msg=to_native(he)) + + if before_rcg or after_rcg: + self.changed = True + if self.module._diff: + self.diff["before"].update(role_config_groups=before_rcg) + self.diff["after"].update(role_config_groups=after_rcg) + + # Handle role override assignments (argspec enforces inclusion of cluster) + # if self.roles or self.purge: + if self.roles: + # if self.roles is None: + # self.roles = list() + + try: + (before_role, after_role) = reconcile_host_role_configs( + api_client=self.api_client, + host=current, + role_configs=self.roles, + purge=self.purge, + check_mode=self.module.check_mode, + message=self.message, + ) + except HostException as he: + self.module.fail_json(msg=to_native(he)) + + if before_role or after_role: + self.changed = True + if self.module._diff: + self.diff["before"].update(roles=before_role) + self.diff["after"].update(roles=after_role) + + # Handle host role states + # TODO Examine to make sure they exit if no cluster or roles to exec upon + if self.state in ["started", "stopped", "restarted"]: + (before_state, after_state) = toggle_host_role_states( + api_client=self.api_client, + host=current, + state=self.state, + check_mode=self.module.check_mode, + ) + + if before_state or after_state: + self.changed = True + if self.module._diff: + self.diff["before"].update(roles=before_state) + self.diff["after"].update(roles=after_state) + + # Refresh if state has changed + if self.changed: + self.output = parse_host_result( + get_host( + api_client=self.api_client, + host_id=current.host_id, + view="full", + ) + ) + else: + self.output = parse_host_result(current) + else: - self.output = parse_host_result(current) + self.module.fail_json(msg="Unknown host state: " + self.state) + + def handle_maintenance(self, host: ApiHost) -> None: + if self.maintenance is not None: + try: + state_changed = toggle_host_maintenance( + api_client=self.api_client, + host=host, + maintenance=self.maintenance, + check_mode=self.module.check_mode, + ) + except HostMaintenanceStateException as ex: + self.module.fail_json(msg=to_native(ex)) + + if state_changed: + self.changed = True + if self.module._diff: + self.diff["before"].update(maintenance_mode=host.maintenance_mode) + self.diff["after"].update(maintenance_mode=self.maintenance) def main(): - module = ClouderaManagerModule.ansible_module( + module = ClouderaManagerMutableModule.ansible_module( argument_spec=dict( name=dict(aliases=["cluster_hostname"]), cluster=dict(aliases=["cluster_name"]), @@ -427,18 +681,34 @@ def main(): maintenance=dict(type="bool", aliases=["maintenance_mode"]), state=dict( default="present", - choices=["present", "absent", "attached", "detached"], + choices=[ + "present", + "absent", + "attached", + "detached", + "started", + "stopped", + "restarted", + ], ), ), required_one_of=[ ("name", "host_id"), ], + mutually_exclusive=[ + ["host_template", "role_config_groups"], + ], required_if=[ ("state", "attached", ("cluster",), False), - # ("state", "attached", ("name", "ip_address",), False), - # ("state", "detached", ("name",), False), - # ("state", "present", ("ip_address",), False), # TODO Move to execution check + ("state", "started", ("cluster",), False), + ("state", "stopped", ("cluster",), False), + ("state", "restarted", ("cluster",), False), ], + required_by={ + "host_template": "cluster", + "role_config_groups": "cluster", + "roles": "cluster", + }, supports_check_mode=True, ) From 603f9bd080fabf51c1a90c07e650cfd8ef46b1b2 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:51 -0400 Subject: [PATCH 09/30] Add shared pytest utilities for 'host' module testing Signed-off-by: Webster Mudge --- tests/unit/plugins/modules/host/conftest.py | 304 ++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 tests/unit/plugins/modules/host/conftest.py diff --git a/tests/unit/plugins/modules/host/conftest.py b/tests/unit/plugins/modules/host/conftest.py new file mode 100644 index 00000000..b0520f1f --- /dev/null +++ b/tests/unit/plugins/modules/host/conftest.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import pytest + +from collections.abc import Callable, Generator +from pathlib import Path + +from cm_client import ( + ApiConfigList, + ApiHost, + ApiHostRef, + ApiHostRefList, + ApiHostRef, + ApiRole, + ApiRoleList, + ApiService, + ClustersResourceApi, + HostsResourceApi, + RolesResourceApi, + ServicesResourceApi, +) + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + wait_commands, + TagUpdates, + ConfigListUpdates, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + read_roles, +) +from ansible_collections.cloudera.cluster.tests.unit import ( + deregister_service, + register_service, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture() +def detached_hosts(cm_api_client) -> list[ApiHost]: + return [ + h + for h in HostsResourceApi(cm_api_client).read_hosts().items + if h.cluster_ref is None + ] + + +@pytest.fixture() +def attached_hosts(cm_api_client, base_cluster) -> list[ApiHost]: + return ( + ClustersResourceApi(cm_api_client) + .list_hosts(cluster_name=base_cluster.name) + .items + ) + + +@pytest.fixture() +def available_hosts(cm_api_client, attached_hosts) -> list[ApiHost]: + return [ + host + for host in attached_hosts + if not HostsResourceApi(cm_api_client).read_host(host_id=host.host_id).role_refs + ] + + +@pytest.fixture(scope="module") +def zookeeper(cm_api_client, base_cluster, request) -> Generator[ApiService]: + # Keep track of the provisioned service(s) + service_registry = list[ApiService]() + + # Get the current cluster hosts + hosts = get_cluster_hosts(cm_api_client, base_cluster) + + id = Path(request.node.parent.name).stem + + zk_service = ApiService( + name=f"test-zk-{id}", + type="ZOOKEEPER", + display_name=f"ZooKeeper ({id})", + # Add a SERVER role (so we can start the service -- a ZK requirement!) + roles=[ApiRole(type="SERVER", host_ref=ApiHostRef(hosts[0].host_id))], + ) + + # Provision and yield the created service + yield register_service( + api_client=cm_api_client, + registry=service_registry, + cluster=base_cluster, + service=zk_service, + ) + + # Remove the created service + deregister_service(api_client=cm_api_client, registry=service_registry) + + +# TODO Split into a new module and scope to its functions +@pytest.fixture(autouse=True) +def resettable_cluster(cm_api_client, base_cluster): + host_api = HostsResourceApi(cm_api_client) + cluster_api = ClustersResourceApi(cm_api_client) + service_api = ServicesResourceApi(cm_api_client) + role_api = RolesResourceApi(cm_api_client) + + # Keep track of attached hosts and their role assignments + prior_hosts = dict[str, (ApiHost, dict[str, ApiRole])]() + + # Get all services on the cluster + prior_services = service_api.read_services( + cluster_name=base_cluster.name, + ).items + + # For each host in the cluster, get a map of each service role type's instance + for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): + prior_roles_by_service = dict[str, dict[str, ApiRole]]() + + # And for each service in the cluster + for s in prior_services: + # Retrieve any roles for the host + prior_roles_by_service[s.name] = { + r.type: r + for r in read_roles( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=s.name, + host_id=h.host_id, + ).items + } + + # Add to the map of prior hosts + prior_hosts[h.host_id] = (h, prior_roles_by_service) + + # yield to the tests + yield base_cluster + + # Each current host + for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): + # If new, remove + if h.host_id not in prior_hosts: + cluster_api.remove_host( + cluster_name=base_cluster.name, + host_id=h.host_id, + ) + # Else, update host, host config, and roles + else: + (prior_host, prior_roles_by_service) = prior_hosts.pop(h.host_id) + host_api.update_host( + host_id=h.host_id, + body=prior_host, + ) + host_api.update_host_config( + host_id=h.host_id, + body=prior_host.config, + ) + + # Get current roles for the host by service + for s in prior_services: + current_roles = read_roles( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=s.name, + host_id=h.host_id, + ).items + + # Retrieve the map of prior service roles (by type) + prior_role_types = prior_roles_by_service.get(s.name) + + for current_role in current_roles: + # If the current has a new role type, remove it + if current_role.type not in prior_role_types: + # Wait for any active commands + active_cmds = role_api.list_active_commands( + cluster_name=base_cluster.name, + service_name=s.name, + role_name=current_role.name, + ) + wait_commands( + api_client=cm_api_client, + commands=active_cmds, + ) + # Delete the role + role_api.delete_role( + cluster_name=base_cluster.name, + service_name=s.name, + role_name=current_role.name, + ) + # Else, update the role and its config with the prior settings + else: + prior_role = prior_role_types.pop(current_role.type) + + if not prior_role.config: + prior_role.config = ApiConfigList() + + role_api.update_role_config( + cluster_name=base_cluster.name, + service_name=s.name, + role_name=current_role.name, + body=prior_role.config, + ) + + # If a prior role type is missing, restore + if prior_role_types: + for r in prior_role_types: + role_api.create_roles( + cluster_name=base_cluster.name, + service_name=r.service_ref.service_name, + body=ApiRoleList(items=[r]), + ) + + # If missing, restore host and roles + if prior_hosts: + cluster_api.add_hosts( + cluster_name=base_cluster.name, + body=ApiHostRefList( + items=[ + ApiHostRef(host_id=prior_host.host_id, hostname=prior_host.hostname) + for prior_host in prior_hosts + ] + ), + ) + + +@pytest.fixture() +def resettable_host(cm_api_client, request) -> Generator[Callable[[ApiHost], ApiHost]]: + host_api = HostsResourceApi(cm_api_client) + + # Registry of resettable hosts + registry = list[ApiHost]() + + # Yield the host wrapper to the tests + def _wrapper(host: ApiHost) -> ApiHost: + registry.append(host) + return host + + yield _wrapper + + # Get current set of hosts + current_hosts_map = dict[str, ApiHost]() + for host in host_api.read_hosts(view="full").items: + host.config = host_api.read_host_config(host.host_id) + current_hosts_map[host.host_id] = host + + # Reset each host + for h in registry: + target_host = current_hosts_map.get(h.host_id, None) + + # If the host was deleted, recreate + if target_host is None: + # TODO Handle creation + pass + else: + # Tags + tag_updates = TagUpdates( + target_host.tags, {t.name: t.value for t in h.tags}, True + ) + if tag_updates.deletions: + host_api.delete_tags( + hostname=target_host.hostname, + body=tag_updates.deletions, + ) + + if tag_updates.additions: + host_api.add_tags( + hostname=target_host.hostname, + body=tag_updates.additions, + ) + + # Config + if h.config is None: + h.config = ApiConfigList(items=[]) + + config_updates = ConfigListUpdates( + target_host.config, {c.name: c.value for c in h.config.items}, True + ) + host_api.update_host_config( + host_id=target_host.host_id, + message=f"{Path(request.node.parent.name).stem}::{request.node.name}", + body=config_updates.config, + ) + + # Cluster + + # Roles From 0d07a603aa1e2ac1a4b058027ee57f58689e4757 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:52 -0400 Subject: [PATCH 10/30] Add tests for host cluster attach/detach functions Signed-off-by: Webster Mudge --- .../modules/host/test_host_clusters.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/unit/plugins/modules/host/test_host_clusters.py diff --git a/tests/unit/plugins/modules/host/test_host_clusters.py b/tests/unit/plugins/modules/host/test_host_clusters.py new file mode 100644 index 00000000..85c331f4 --- /dev/null +++ b/tests/unit/plugins/modules/host/test_host_clusters.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import pytest + +from collections.abc import Callable, Generator +from pathlib import Path + +from cm_client import ( + ApiConfig, + ApiConfigList, + ApiEntityTag, + ApiHost, + ApiHostList, + ApiHostRef, + ApiHostRefList, + ApiHostTemplate, + ApiHostTemplateList, + ApiHostRef, + ApiRole, + ApiRoleConfigGroup, + ApiRoleConfigGroupRef, + ApiRoleList, + ApiService, + ClouderaManagerResourceApi, + ClustersResourceApi, + HostsResourceApi, + HostTemplatesResourceApi, + RolesResourceApi, + ServicesResourceApi, +) +from cm_client.rest import ApiException + +from ansible_collections.cloudera.cluster.plugins.modules import host +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + wait_commands, + TagUpdates, + ConfigListUpdates, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + create_role, + provision_service_role, + read_roles, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + create_host_template_model, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( + get_host_roles, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, + deregister_service, + register_service, +) + +LOG = logging.getLogger(__name__) + + +class TestHostAttached: + def test_host_create_invalid_cluster(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() + + +class TestHostDetached: + def test_host_create_invalid_cluster(self, conn, module_args): + module_args( + { + **conn, + } + ) + + with pytest.raises(AnsibleFailJson, match="boom") as e: + host.main() From 25caa7357909016d472b0e9543d10f6d11cdf5a5 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:53 -0400 Subject: [PATCH 11/30] Add tests for host template assignments and bi-directional reconciliation Signed-off-by: Webster Mudge --- .../modules/host/test_host_host_templates.py | 345 ++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 tests/unit/plugins/modules/host/test_host_host_templates.py diff --git a/tests/unit/plugins/modules/host/test_host_host_templates.py b/tests/unit/plugins/modules/host/test_host_host_templates.py new file mode 100644 index 00000000..cae56718 --- /dev/null +++ b/tests/unit/plugins/modules/host/test_host_host_templates.py @@ -0,0 +1,345 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import pytest + +from pathlib import Path + +from cm_client import ( + HostsResourceApi, +) + +from ansible_collections.cloudera.cluster.plugins.modules import host + +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + create_role, + provision_service_role, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( + create_host_template_model, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( + get_host_roles, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + get_base_role_config_group, +) +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, +) + +LOG = logging.getLogger(__name__) + + +class TestHostHostTemplates: + def test_host_update_host_template_new( + self, + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + available_hosts, + host_template_factory, + ): + target_name = f"pytest-{Path(request.node.name)}" + + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Get the base RCG for ZK SERVER + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Create a host template and assign the base ZK SERVER RCG + host_template = host_template_factory( + cluster=base_cluster, + host_template=create_host_template_model( + cluster_name=base_cluster.name, + name=target_name, + role_config_groups=[target_rcg], + ), + ) + + # Set the host template + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": target_host.cluster_ref.cluster_name, + "host_template": host_template.name, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + def test_host_update_host_template_existing( + self, + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + available_hosts, + host_template_factory, + ): + target_name = f"pytest-{Path(request.node.name)}" + + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Get the base RCG for ZK SERVER + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Create a host template and assign the base ZK SERVER RCG + host_template = host_template_factory( + cluster=base_cluster, + host_template=create_host_template_model( + cluster_name=base_cluster.name, + name=target_name, + role_config_groups=[target_rcg], + ), + ) + + # Set the host template + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": target_host.cluster_ref.cluster_name, + "host_template": host_template.name, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set( + [ + target_rcg.name, + existing_role.role_config_group_ref.role_config_group_name, + ] + ) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set( + [ + target_rcg.name, + existing_role.role_config_group_ref.role_config_group_name, + ] + ) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + def test_host_update_host_template_purge( + self, + conn, + module_args, + request, + cm_api_client, + base_cluster, + zookeeper, + available_hosts, + host_template_factory, + ): + target_name = f"pytest-{Path(request.node.name)}" + + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Get the base RCG for ZK SERVER + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Create a host template and assign the base ZK SERVER RCG + host_template = host_template_factory( + cluster=base_cluster, + host_template=create_host_template_model( + cluster_name=base_cluster.name, + name=target_name, + role_config_groups=[target_rcg], + ), + ) + + # Set the host template + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": target_host.cluster_ref.cluster_name, + "host_template": host_template.name, + "purge": True, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) From 07e1c3b9f1e8341e5cbdc67755281771ce6af3ef Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:53 -0400 Subject: [PATCH 12/30] Add tests for role config group assignments and reconciliation for hosts Signed-off-by: Webster Mudge --- .../plugins/modules/host/test_host_rcgs.py | 651 ++++++++++++++++++ 1 file changed, 651 insertions(+) create mode 100644 tests/unit/plugins/modules/host/test_host_rcgs.py diff --git a/tests/unit/plugins/modules/host/test_host_rcgs.py b/tests/unit/plugins/modules/host/test_host_rcgs.py new file mode 100644 index 00000000..8871f7d0 --- /dev/null +++ b/tests/unit/plugins/modules/host/test_host_rcgs.py @@ -0,0 +1,651 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import pytest + +from pathlib import Path + +from cm_client import ( + HostsResourceApi, +) + +from ansible_collections.cloudera.cluster.plugins.modules import host + +from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( + get_host_roles, +) + +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + create_role, + provision_service_role, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( + create_role_config_group, + get_base_role_config_group, +) + +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +class TestHostRoleConfigGroups: + def test_host_update_role_config_group_invalid_service( + self, conn, module_args, available_hosts, zookeeper + ): + + target_host = available_hosts[0] + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": "BOOM", + "type": "Example", + }, + ], + } + ) + + with pytest.raises(AnsibleFailJson, match="Service 'BOOM' not found"): + host.main() + + def test_host_update_role_config_group_invalid_type( + self, conn, module_args, available_hosts, zookeeper + ): + target_host = available_hosts[0] + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "type": "BOOM", + }, + ], + } + ) + + with pytest.raises( + AnsibleFailJson, match="Base role config group for type, 'BOOM', not found" + ): + host.main() + + def test_host_update_role_config_group_invalid_name( + self, + conn, + module_args, + cm_api_client, + available_hosts, + zookeeper, + role_config_group_factory, + request, + ): + id = f"pytest-{Path(request.node.name).stem}" + + role_config_group_factory( + service=zookeeper, + role_config_group=create_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + name=id, + role_type="SERVER", + ), + ) + + target_host = available_hosts[0] + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": "BOOM", + }, + ], + } + ) + + with pytest.raises( + AnsibleFailJson, match="The role config group 'BOOM' does not exist" + ): + host.main() + + def test_host_update_role_config_group_new_name( + self, + conn, + module_args, + cm_api_client, + available_hosts, + zookeeper, + role_config_group_factory, + request, + ): + id = f"pytest-{Path(request.node.name).stem}" + + target_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=create_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + name=id, + role_type="SERVER", + ), + ) + + # Target a host without ZK Server + target_host = available_hosts[0] + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + def test_host_update_role_config_group_new_base( + self, conn, module_args, cm_api_client, available_hosts, zookeeper + ): + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Target a host without ZK Server + target_host = available_hosts[0] + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + def test_host_update_role_config_group_existing_name( + self, + conn, + module_args, + cm_api_client, + available_hosts, + base_cluster, + zookeeper, + role_config_group_factory, + request, + ): + id = f"pytest-{Path(request.node.name).stem}" + + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Create a custom RCG for ZK SERVER + target_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=create_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + name=id, + role_type="SERVER", + ), + ) + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + def test_host_update_role_config_group_existing_base( + self, conn, module_args, cm_api_client, available_hosts, base_cluster, zookeeper + ): + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Get the base RCG for ZK SERVER + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert target_rcg.name in [ + role.role_config_group_ref.role_config_group_name for role in current_roles + ] + + def test_host_update_role_config_group_purge_name( + self, + conn, + module_args, + cm_api_client, + available_hosts, + base_cluster, + zookeeper, + role_config_group_factory, + request, + ): + id = f"pytest-{Path(request.node.name).stem}" + + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Create a custom RCG for ZK SERVER + target_rcg = role_config_group_factory( + service=zookeeper, + role_config_group=create_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + name=id, + role_type="SERVER", + ), + ) + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + "purge": True, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + def test_host_update_role_config_group_purge_base( + self, conn, module_args, cm_api_client, available_hosts, base_cluster, zookeeper + ): + # Get an existing, non-ZK SERVER host + target_host = available_hosts[0] + + # Add an existing role to the target host + existing_role = create_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role_type="GATEWAY", + host_id=target_host.host_id, + ) + + # Provision the existing role to the target host + existing_role = provision_service_role( + api_client=cm_api_client, + cluster_name=base_cluster.name, + service_name=zookeeper.name, + role=existing_role, + ) + + # Get the base RCG for ZK SERVER + target_rcg = get_base_role_config_group( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + ) + + # Set the role config + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": zookeeper.cluster_ref.cluster_name, + "role_config_groups": [ + { + "service": zookeeper.name, + "name": target_rcg.name, + }, + ], + "purge": True, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + # Reread the host + updated_host = HostsResourceApi(cm_api_client).read_host( + host_id=target_host.host_id + ) + + # Retrieve the current running roles on the host + current_roles = get_host_roles(api_client=cm_api_client, host=updated_host) + assert set(e.value.host["roles"]) == set([role.name for role in current_roles]) + assert set([target_rcg.name]) == set( + [ + role.role_config_group_ref.role_config_group_name + for role in current_roles + ] + ) From 2b272a16335e8edd0a4499e56c11ad2e58ce0a74 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:54 -0400 Subject: [PATCH 13/30] Add tests for role configuration overrides and reconciliation for hosts Signed-off-by: Webster Mudge --- .../modules/host/test_host_role_configs.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 tests/unit/plugins/modules/host/test_host_role_configs.py diff --git a/tests/unit/plugins/modules/host/test_host_role_configs.py b/tests/unit/plugins/modules/host/test_host_role_configs.py new file mode 100644 index 00000000..b7daccd7 --- /dev/null +++ b/tests/unit/plugins/modules/host/test_host_role_configs.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import pytest + +from ansible_collections.cloudera.cluster.plugins.modules import host + +from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( + create_role, + read_role, +) + +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +class TestHostRoleConfigs: + def test_host_update_role_config_invalid_type( + self, conn, module_args, cm_api_client, available_hosts, zookeeper, role_factory + ): + role_model = create_role( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + host_id=available_hosts[0].host_id, + ) + + existing_role = role_factory( + service=zookeeper, + role=role_model, + ) + + # Set the role config + module_args( + { + **conn, + "name": existing_role.host_ref.hostname, + "cluster": existing_role.service_ref.cluster_name, + "roles": [ + { + "service": existing_role.service_ref.service_name, + "type": "BOOM", + "config": { + "maxSessionTimeout": 50001, + "process_start_secs": 31, + }, + }, + ], + } + ) + + with pytest.raises(AnsibleFailJson, match="No role of type, 'BOOM'"): + host.main() + + def test_host_update_role_config( + self, conn, module_args, cm_api_client, available_hosts, zookeeper, role_factory + ): + role_model = create_role( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + host_id=available_hosts[0].host_id, + config={ + "minSessionTimeout": 5001, + "maxSessionTimeout": 40000, + }, + ) + + existing_role = role_factory( + service=zookeeper, + role=role_model, + ) + + # Set the role config + module_args( + { + **conn, + "name": existing_role.host_ref.hostname, + "cluster": existing_role.service_ref.cluster_name, + "roles": [ + { + "service": existing_role.service_ref.service_name, + "type": existing_role.type, + "config": { + "maxSessionTimeout": 50001, + "process_start_secs": 31, + }, + }, + ], + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + updated_role = read_role( + api_client=cm_api_client, + cluster_name=existing_role.service_ref.cluster_name, + service_name=existing_role.service_ref.service_name, + role_name=existing_role.name, + ) + + assert ( + dict( + minSessionTimeout="5001", + maxSessionTimeout="50001", + process_start_secs="31", + ).items() + <= {c.name: c.value for c in updated_role.config.items}.items() + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + updated_role = read_role( + api_client=cm_api_client, + cluster_name=existing_role.service_ref.cluster_name, + service_name=existing_role.service_ref.service_name, + role_name=existing_role.name, + ) + + assert ( + dict( + minSessionTimeout="5001", + maxSessionTimeout="50001", + process_start_secs="31", + ).items() + <= {c.name: c.value for c in updated_role.config.items}.items() + ) + + def test_host_update_role_config_purge( + self, conn, module_args, cm_api_client, available_hosts, zookeeper, role_factory + ): + + role_model = create_role( + api_client=cm_api_client, + cluster_name=zookeeper.cluster_ref.cluster_name, + service_name=zookeeper.name, + role_type="SERVER", + host_id=available_hosts[0].host_id, + config={ + "minSessionTimeout": 5001, + "maxSessionTimeout": 40000, + }, + ) + + existing_role = role_factory( + service=zookeeper, + role=role_model, + ) + + # Set the role config + module_args( + { + **conn, + "name": existing_role.host_ref.hostname, + "cluster": existing_role.service_ref.cluster_name, + "roles": [ + { + "service": existing_role.service_ref.service_name, + "type": existing_role.type, + "config": { + "minSessionTimeout": 5001, + "process_start_secs": 31, + }, + }, + ], + "purge": True, + "cluster": existing_role.service_ref.cluster_name, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + + updated_config = { + c.name: c.value + for c in read_role( + api_client=cm_api_client, + cluster_name=existing_role.service_ref.cluster_name, + service_name=existing_role.service_ref.service_name, + role_name=existing_role.name, + ).config.items + } + + assert ( + dict( + minSessionTimeout="5001", + process_start_secs="31", + ).items() + <= updated_config.items() + ) + assert "maxSessionTimeout" not in updated_config + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + + updated_config = { + c.name: c.value + for c in read_role( + api_client=cm_api_client, + cluster_name=existing_role.service_ref.cluster_name, + service_name=existing_role.service_ref.service_name, + role_name=existing_role.name, + ).config.items + } + + assert ( + dict( + minSessionTimeout="5001", + process_start_secs="31", + ).items() + <= updated_config.items() + ) + assert "maxSessionTimeout" not in updated_config From 2a37f9a6d8dd19a7a35353261ba3416cbbc6ea63 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:55 -0400 Subject: [PATCH 14/30] Update tests for core host management and configuration Signed-off-by: Webster Mudge --- tests/unit/plugins/modules/host/test_host.py | 619 ++++++++++--------- 1 file changed, 310 insertions(+), 309 deletions(-) diff --git a/tests/unit/plugins/modules/host/test_host.py b/tests/unit/plugins/modules/host/test_host.py index dd3800a6..9a271834 100644 --- a/tests/unit/plugins/modules/host/test_host.py +++ b/tests/unit/plugins/modules/host/test_host.py @@ -25,213 +25,23 @@ from pathlib import Path from cm_client import ( + ApiConfig, ApiConfigList, + ApiEntityTag, ApiHost, - ApiHostList, - ApiHostRef, - ApiHostRefList, - ApiHostTemplate, - ApiHostTemplateList, - ApiHostRef, - ApiRole, - ApiRoleConfigGroup, - ApiRoleConfigGroupRef, - ApiRoleList, - ApiService, - ClouderaManagerResourceApi, - ClustersResourceApi, HostsResourceApi, - HostTemplatesResourceApi, - RolesResourceApi, - ServicesResourceApi, ) -from cm_client.rest import ApiException from ansible_collections.cloudera.cluster.plugins.modules import host -from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( - get_cluster_hosts, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( - read_roles, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( - create_host_template_model, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( - get_base_role_config_group, -) + from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, - deregister_service, - register_service, ) LOG = logging.getLogger(__name__) -@pytest.fixture() -def available_hosts(cm_api_client) -> list[ApiHost]: - return [ - h - for h in HostsResourceApi(cm_api_client).read_hosts().items - if h.cluster_ref is None - ] - - -@pytest.fixture() -def cluster_hosts(cm_api_client, base_cluster) -> list[ApiHost]: - return ( - ClustersResourceApi(cm_api_client) - .list_hosts(cluster_name=base_cluster.name) - .items - ) - - -@pytest.fixture(scope="module") -def zookeeper(cm_api_client, base_cluster, request) -> Generator[ApiService]: - # Keep track of the provisioned service(s) - service_registry = list[ApiService]() - - # Get the current cluster hosts - hosts = get_cluster_hosts(cm_api_client, base_cluster) - - id = Path(request.node.parent.name).stem - - zk_service = ApiService( - name=f"test-zk-{id}", - type="ZOOKEEPER", - display_name=f"ZooKeeper ({id})", - # Add a SERVER role (so we can start the service -- a ZK requirement!) - roles=[ApiRole(type="SERVER", host_ref=ApiHostRef(hosts[0].host_id))], - ) - - # Provision and yield the created service - yield register_service( - api_client=cm_api_client, - registry=service_registry, - cluster=base_cluster, - service=zk_service, - ) - - # Remove the created service - deregister_service(api_client=cm_api_client, registry=service_registry) - - -@pytest.fixture(autouse=True) -def resettable_cluster(cm_api_client, base_cluster): - host_api = HostsResourceApi(cm_api_client) - cluster_api = ClustersResourceApi(cm_api_client) - service_api = ServicesResourceApi(cm_api_client) - role_api = RolesResourceApi(cm_api_client) - - # Keep track of attached hosts and their role assignments - prior_hosts = dict[str, (ApiHost, dict[str, ApiRole])]() - - # Get all services on the cluster - prior_services = service_api.read_services( - cluster_name=base_cluster.name, - ).items - - # For each host in the cluster, get a map of each service role type's instance - for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): - prior_roles_by_service = dict[str, dict[str, ApiRole]]() - - # And for each service in the cluster - for s in prior_services: - # Retrieve any roles for the host - prior_roles_by_service[s.name] = { - r.type: r - for r in read_roles( - api_client=cm_api_client, - cluster_name=base_cluster.name, - service_name=s.name, - host_id=h.host_id, - ).items - } - - # Add to the map of prior hosts - prior_hosts[h.host_id] = (h, prior_roles_by_service) - - # yield to the tests - yield base_cluster - - # Each current host - for h in get_cluster_hosts(api_client=cm_api_client, cluster=base_cluster): - # If new, remove - if h.host_id not in prior_hosts: - cluster_api.remove_host( - cluster_name=base_cluster.name, - host_id=h.host_id, - ) - # Else, update host, host config, and roles - else: - (prior_host, prior_roles_by_service) = prior_hosts.pop(h.host_id) - host_api.update_host( - host_id=h.host_id, - body=prior_host, - ) - host_api.update_host_config( - host_id=h.host_id, - body=prior_host.config, - ) - - # Get current roles for the host by service - for s in prior_services: - current_roles = read_roles( - api_client=cm_api_client, - cluster_name=base_cluster.name, - service_name=s.name, - host_id=h.host_id, - ).items - - # Retrieve the map of prior service roles (by type) - prior_role_types = prior_roles_by_service.get(s.name) - - for current_role in current_roles: - # If the current has a new role type, remove it - if current_role.type not in prior_role_types: - role_api.delete_role( - cluster_name=base_cluster.name, - service_name=s.name, - role_name=current_role.name, - ) - # Else, update the role and its config with the prior settings - else: - prior_role = prior_role_types.pop(current_role.type) - - if not prior_role.config: - prior_role.config = ApiConfigList() - - role_api.update_role_config( - cluster_name=base_cluster.name, - service_name=s.name, - role_name=current_role.name, - body=prior_role.config, - ) - - # If a prior role type is missing, restore - if prior_role_types: - for r in prior_role_types: - role_api.create_roles( - cluster_name=base_cluster.name, - service_name=r.service_ref.service_name, - body=ApiRoleList(items=[r]), - ) - - # If missing, restore host and roles - if prior_hosts: - cluster_api.add_hosts( - cluster_name=base_cluster.name, - body=ApiHostRefList( - items=[ - ApiHostRef(host_id=prior_host.host_id, hostname=prior_host.hostname) - for prior_host in prior_hosts - ] - ), - ) - - class TestHostArgSpec: def test_host_missing_required(self, conn, module_args): module_args( @@ -245,7 +55,7 @@ def test_host_missing_required(self, conn, module_args): ) as e: host.main() - def test_host_missing_attached_ip_address(self, conn, module_args): + def test_host_missing_attached_cluster(self, conn, module_args): module_args( { **conn, @@ -260,6 +70,61 @@ def test_host_missing_attached_ip_address(self, conn, module_args): ) as e: host.main() + def test_host_missing_host_template_cluster(self, conn, module_args): + module_args( + { + **conn, + "name": "example", + "host_template": "example", + } + ) + + with pytest.raises( + AnsibleFailJson, + match="required by 'host_template': cluster", + ) as e: + host.main() + + def test_host_missing_role_config_groups_cluster(self, conn, module_args): + module_args( + { + **conn, + "name": "example", + "role_config_groups": [ + { + "service": "example", + "type": "example", + }, + ], + } + ) + + with pytest.raises( + AnsibleFailJson, + match="required by 'role_config_groups': cluster", + ) as e: + host.main() + + def test_host_missing_roles_cluster(self, conn, module_args): + module_args( + { + **conn, + "name": "example", + "roles": [ + { + "service": "example", + "type": "example", + }, + ], + } + ) + + with pytest.raises( + AnsibleFailJson, + match="required by 'roles': cluster", + ) as e: + host.main() + # TODO Tackle the mutations first, as provisioning will require a host without CM agent... @pytest.mark.skip() @@ -278,12 +143,12 @@ def test_host_create_missing_ip_address(self, conn, module_args): ) as e: host.main() - def test_host_create_ip_address(self, conn, module_args, available_hosts): + def test_host_create_ip_address(self, conn, module_args, detached_hosts): module_args( { **conn, "name": "pytest-host", - "ip_address": available_hosts[0].ip_address, + "ip_address": detached_hosts[0].ip_address, } ) @@ -332,8 +197,46 @@ def test_host_create_maintenance_enabled(self, conn, module_args): class TestHostModification: - def test_host_update_ip_address(self, conn, module_args, cluster_hosts): - target_host = cluster_hosts[0] + @pytest.fixture() + def maintenance_enabled_host( + self, cm_api_client, detached_hosts + ) -> Generator[ApiHost]: + target_host = detached_hosts[0] + + # Set the host to maintenance mode if not already set + if not target_host.maintenance_mode: + HostsResourceApi(cm_api_client).enter_maintenance_mode(target_host.host_id) + + # Yield to the test + yield target_host + + # Reset the maintenance mode + if target_host.maintenance_mode: + HostsResourceApi(cm_api_client).enter_maintenance_mode(target_host.host_id) + else: + HostsResourceApi(cm_api_client).exit_maintenance_mode(target_host.host_id) + + @pytest.fixture() + def maintenance_disabled_host( + self, cm_api_client, detached_hosts + ) -> Generator[ApiHost]: + target_host = detached_hosts[0] + + # Unset the host to maintenance mode if not already set + if target_host.maintenance_mode: + HostsResourceApi(cm_api_client).exit_maintenance_mode(target_host.host_id) + + # Yield to the test + yield target_host + + # Reset the maintenance mode + if target_host.maintenance_mode: + HostsResourceApi(cm_api_client).enter_maintenance_mode(target_host.host_id) + else: + HostsResourceApi(cm_api_client).exit_maintenance_mode(target_host.host_id) + + def test_host_update_ip_address(self, conn, module_args, attached_hosts): + target_host = attached_hosts[0] module_args( { @@ -346,8 +249,8 @@ def test_host_update_ip_address(self, conn, module_args, cluster_hosts): with pytest.raises(AnsibleFailJson, match="To update the host IP address") as e: host.main() - def test_host_update_rack_id(self, conn, module_args, cluster_hosts): - target_host = cluster_hosts[0] + def test_host_update_rack_id(self, conn, module_args, attached_hosts): + target_host = attached_hosts[0] module_args( { @@ -370,41 +273,37 @@ def test_host_update_rack_id(self, conn, module_args, cluster_hosts): assert e.value.changed == False assert e.value.host["rack_id"] == "/pytest1" - def test_host_update_host_template( + def test_host_update_tags( self, conn, module_args, - request, cm_api_client, - base_cluster, - zookeeper, - cluster_hosts, - role_config_group_factory, - host_template_factory, + detached_hosts, + resettable_host, ): - target_host = cluster_hosts[0] - target_name = f"pytest-{Path(request.node.name)}" - target_rcg = get_base_role_config_group( - api_client=cm_api_client, - cluster_name=zookeeper.cluster_ref.cluster_name, - service_name=zookeeper.name, - role_type="SERVER", - ) - - host_template = host_template_factory( - cluster=base_cluster, - host_template=create_host_template_model( - cluster_name=base_cluster.name, - name=target_name, - role_config_groups=[target_rcg], - ), + HostsResourceApi(cm_api_client) + + # Get a detached host + target_host = resettable_host(detached_hosts[0]) + + # Update the host's tags + HostsResourceApi(cm_api_client).add_tags( + hostname=target_host.hostname, + body=[ + ApiEntityTag(name="tag_one", value="Existing"), + ApiEntityTag(name="tag_two", value="Existing"), + ], ) + # Set the tags module_args( { **conn, "name": target_host.hostname, - "host_template": host_template.name, + "tags": { + "tag_one": "Updated", + "tag_three": "Added", + }, } ) @@ -412,142 +311,244 @@ def test_host_update_host_template( host.main() assert e.value.changed == True + assert e.value.host["tags"] == dict( + tag_one="Updated", tag_two="Existing", tag_three="Added" + ) # Idempotency with pytest.raises(AnsibleExitJson) as e: host.main() assert e.value.changed == False + assert e.value.host["tags"] == dict( + tag_one="Updated", tag_two="Existing", tag_three="Added" + ) + + def test_host_update_tags_purge( + self, + conn, + module_args, + cm_api_client, + detached_hosts, + resettable_host, + ): + HostsResourceApi(cm_api_client) + + # Get a detached host + target_host = resettable_host(detached_hosts[0]) + + # Update the host's tags + HostsResourceApi(cm_api_client).add_tags( + hostname=target_host.hostname, + body=[ + ApiEntityTag(name="tag_one", value="Existing"), + ApiEntityTag(name="tag_two", value="Existing"), + ], + ) - def test_host_update_host_template_new_role(self, conn, module_args): + # Set the tags module_args( { **conn, + "name": target_host.hostname, + "tags": { + "tag_one": "Updated", + "tag_three": "Added", + }, + # Note that if using an attached host, be sure to include the cluster name + # or purge will detach the host from the cluster! + "purge": True, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + assert e.value.host["tags"] == dict(tag_one="Updated", tag_three="Added") + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: host.main() - def test_host_update_tags(self, conn, module_args): + assert e.value.changed == False + assert e.value.host["tags"] == dict(tag_one="Updated", tag_three="Added") + + def test_host_update_config( + self, + conn, + module_args, + cm_api_client, + detached_hosts, + resettable_host, + request, + ): + id = Path(request.node.parent.name).stem + HostsResourceApi(cm_api_client) + + # Get a detached host + target_host = resettable_host(detached_hosts[0]) + + # Update the host's tags + HostsResourceApi(cm_api_client).update_host_config( + host_id=target_host.host_id, + message=f"pytest-{id}", + body=ApiConfigList( + items=[ + ApiConfig(name="memory_overcommit_threshold", value="0.85"), + ApiConfig(name="host_memswap_window", value="16"), + ] + ), + ) + + # Set the tags module_args( { **conn, + "name": target_host.hostname, + "config": { + "host_network_frame_errors_window": "20", + "host_memswap_window": "20", + }, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises(AnsibleExitJson) as e: host.main() - def test_host_update_maintenance_enabled(self, conn, module_args): - module_args( - { - **conn, - } + assert e.value.changed == True + assert e.value.host["config"] == dict( + memory_overcommit_threshold="0.85", + host_memswap_window="20", + host_network_frame_errors_window="20", ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + # Idempotency + with pytest.raises(AnsibleExitJson) as e: host.main() - def test_host_update_maintenance_disabled(self, conn, module_args): + assert e.value.changed == False + assert e.value.host["config"] == dict( + memory_overcommit_threshold="0.85", + host_memswap_window="20", + host_network_frame_errors_window="20", + ) + + def test_host_update_config_purge( + self, + conn, + module_args, + cm_api_client, + detached_hosts, + resettable_host, + request, + ): + id = Path(request.node.parent.name).stem + HostsResourceApi(cm_api_client) + + # Get a detached host + target_host = resettable_host(detached_hosts[0]) + + # Update the host's tags + HostsResourceApi(cm_api_client).update_host_config( + host_id=target_host.host_id, + message=f"pytest-{id}", + body=ApiConfigList( + items=[ + ApiConfig(name="memory_overcommit_threshold", value="0.85"), + ApiConfig(name="host_memswap_window", value="16"), + ] + ), + ) + + # Set the tags module_args( { **conn, + "name": target_host.hostname, + "config": { + "host_network_frame_errors_window": "20", + "host_memswap_window": "20", + }, + "purge": True, + # Note that if using an attached host, be sure to set 'cluster' or it will + # be detached due to the 'purge' flag! } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + assert e.value.host["config"] == dict( + host_memswap_window="20", + host_network_frame_errors_window="20", + ) + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: host.main() + assert e.value.changed == False + assert e.value.host["config"] == dict( + host_memswap_window="20", + host_network_frame_errors_window="20", + ) -class TestHostAttached: - def test_host_create_invalid_cluster(self, conn, module_args): + def test_host_update_maintenance_enabled( + self, conn, module_args, maintenance_disabled_host + ): module_args( { **conn, + "name": maintenance_disabled_host.hostname, + "maintenance": True, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises( + AnsibleExitJson, + ) as e: host.main() + assert e.value.changed == True + assert e.value.host["maintenance_mode"] == True -class TestHostDetached: - def test_host_create_invalid_cluster(self, conn, module_args): + # Idempotency + + with pytest.raises( + AnsibleExitJson, + ) as e: + host.main() + + assert e.value.changed == False + assert e.value.host["maintenance_mode"] == True + + def test_host_update_maintenance_disabled( + self, conn, module_args, maintenance_enabled_host + ): module_args( { **conn, + "name": maintenance_enabled_host.hostname, + "maintenance": False, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises( + AnsibleExitJson, + ) as e: host.main() + assert e.value.changed == True + assert e.value.host["maintenance_mode"] == False -# def test_pytest_add_host_to_cloudera_manager(module_args): -# module_args( -# { -# "username": os.getenv("CM_USERNAME"), -# "password": os.getenv("CM_PASSWORD"), -# "host": os.getenv("CM_HOST"), -# "port": "7180", -# "verify_tls": "no", -# "debug": "no", -# "cluster_hostname": "cloudera.host.example", -# "rack_id": "/defo", -# "cluster_host_ip": "10.10.1.1", -# "state": "present", -# } -# ) - -# with pytest.raises(AnsibleExitJson) as e: -# host.main() - -# # LOG.info(str(e.value)) -# LOG.info(str(e.value.cloudera_manager)) - - -# def test_pytest_attach_host_to_cluster(module_args): -# module_args( -# { -# "username": os.getenv("CM_USERNAME"), -# "password": os.getenv("CM_PASSWORD"), -# "host": os.getenv("CM_HOST"), -# "port": "7180", -# "verify_tls": "no", -# "debug": "no", -# "cluster_hostname": "cloudera.host.example", -# "name": "Cluster_Example", -# "rack_id": "/defo", -# "cluster_host_ip": "10.10.1.1", -# "state": "attached", -# } -# ) - -# with pytest.raises(AnsibleExitJson) as e: -# host.main() - -# # LOG.info(str(e.value)) -# LOG.info(str(e.value.cloudera_manager)) - - -# def test_pytest_detach_host_from_cluster(module_args): -# module_args( -# { -# "username": os.getenv("CM_USERNAME"), -# "password": os.getenv("CM_PASSWORD"), -# "host": os.getenv("CM_HOST"), -# "port": "7180", -# "verify_tls": "no", -# "debug": "no", -# "cluster_hostname": "cloudera.host.example", -# "name": "Cluster_Example", -# "state": "detached", -# } -# ) - -# with pytest.raises(AnsibleExitJson) as e: -# host.main() - -# # LOG.info(str(e.value)) -# LOG.info(str(e.value.cloudera_manager)) + # Idempotency + + with pytest.raises( + AnsibleExitJson, + ) as e: + host.main() + + assert e.value.changed == False + assert e.value.host["maintenance_mode"] == False From a189276e26eec8191a50a36b3798b286bc590d82 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:56 -0400 Subject: [PATCH 15/30] Remove obsolete code Signed-off-by: Webster Mudge --- plugins/module_utils/host_utils.py | 95 ------------------------------ 1 file changed, 95 deletions(-) diff --git a/plugins/module_utils/host_utils.py b/plugins/module_utils/host_utils.py index 26ce2836..dee8df0d 100644 --- a/plugins/module_utils/host_utils.py +++ b/plugins/module_utils/host_utils.py @@ -487,21 +487,9 @@ def reconcile_host_template_assignments( check_mode: bool, ) -> tuple[list[dict], list[dict]]: - # diff_before, diff_after = list[dict](), list[dict]() - host_template_api = HostTemplatesResourceApi(api_client) - # service_api = ServicesResourceApi(api_client) - # rcg_api = RoleConfigGroupsResourceApi(api_client) - # role_api = RolesResourceApi(api_client) # Index by all cluster role config groups by name - # cluster_rcg_map = { - # rcg.name: rcg - # for service in service_api.read_services(cluster_name=cluster.name).items - # for rcg in rcg_api.read_role_config_groups( - # cluster_name=cluster.name, service_name=service.name - # ).items - # } cluster_rcg_map = _read_cluster_rcgs( api_client=api_client, cluster=cluster, @@ -520,22 +508,6 @@ def reconcile_host_template_assignments( ) # Retrieve the associated role config groups from each installed role - # current_rcgs = dict[str, ApiRoleConfigGroup]() - # for role_ref in host.role_refs: - # role = read_role( - # api_client=api_client, - # cluster_name=role_ref.cluster_name, - # service_name=role_ref.service_name, - # role_name=role_ref.role_name, - # ) - # if role.role_config_group_ref.role_config_group_name in cluster_rcg_map: - # current_rcgs[ - # role.role_config_group_ref.role_config_group_name - # ] = cluster_rcg_map[role.role_config_group_ref.role_config_group_name] - # else: - # raise Exception( - # f"Invalid role config group reference, '{role.role_config_group_ref.role_config_group_name}', on host, {host.hostname}" - # ) current_rcgs = _read_host_rcgs( api_client=api_client, host=host, @@ -587,73 +559,6 @@ def reconcile_host_template_assignments( purge=purge, check_mode=check_mode, ) - # additions = set(ht_rcgs.keys()) - set(current_rcgs.keys()) - # deletions = set(current_rcgs.keys()) - set(ht_rcgs.keys()) - - # # If the host template has additional assignments - # if additions: - # for add_rcg_name in additions: - - # # Retrieve the role config group by name from the cluster - # add_rcg = cluster_rcg_map[add_rcg_name] - - # # Create the role instance model using the role config group - # created_role = create_role( - # api_client=api_client, - # host_id=host.host_id, - # cluster_name=add_rcg.service_ref.cluster_name, - # service_name=add_rcg.service_ref.service_name, - # role_type=add_rcg.role_type, - # role_config_group=add_rcg.name, - # ) - - # diff_before.append(dict()) - # diff_after.append(created_role.to_dict()) - - # if not check_mode: - # provision_service_role( - # api_client=api_client, - # cluster_name=add_rcg.service_ref.cluster_name, - # service_name=add_rcg.service_ref.service_name, - # role=created_role, - # ) - - # # If the host has extraneous assignments - # if deletions and purge: - # for del_rcg_name in deletions: - - # # Retrieve the current role config group by name - # del_rcg = cluster_rcg_map[del_rcg_name] # current_rcgs[del_rcg_name] - - # # Retrieve the role instance on the host via the role config group's type - # del_roles = read_roles( - # api_client=api_client, - # host_id=host.host_id, - # cluster_name=del_rcg.service_ref.cluster_name, - # service_name=del_rcg.service_ref.service_name, - # type=del_rcg.role_type, - # ).items - - # if not del_roles: - # raise Exception( - # f"Error reading role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" - # ) - # if len(del_roles) != 1: - # raise Exception( - # f"Error, multiple instances for role type, '{del_rcg.role_type}', for service, '{del_rcg.service_ref.service_name}', on cluster, '{del_rcg.service_ref.cluster_name}'" - # ) - - # diff_before.append(del_roles[0].to_dict()) - # diff_after.append(dict()) - - # if not check_mode: - # role_api.delete_role( - # cluster_name=del_roles[0].service_ref.cluster_name, - # service_name=del_roles[0].service_ref.service_name, - # role_name=del_roles[0].name, - # ) - - # return (diff_before, diff_after) def toggle_host_role_states( From d467825df853824ce810bf95f8ae6bb5ee07f396 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:56 -0400 Subject: [PATCH 16/30] Update host module documentation Signed-off-by: Webster Mudge --- plugins/modules/host.py | 358 +++++++++++++++++++++++++++------------- 1 file changed, 242 insertions(+), 116 deletions(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index 665ba332..d4a0be37 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -17,59 +17,156 @@ DOCUMENTATION = r""" module: host -short_description: Manage hosts within Cloudera Manager +short_description: Manage Cloudera Manager hosts description: - - Allows for the management of hosts within the Cloudera Manager. - - It provides functionalities to create, delete, attach, or detach host instance from a cluster. + - Allows for the management of Cloudera Manager hosts. + - Functionality includes creation and deletion of hosts; host cluster assignment, host template, role config group assignment, and host and role instance configuration. author: - "Ronald Suplina (@rsuplina)" -requirements: - - cm_client + - "Webster Mudge (@wmudge)" options: - cluster_hostname: + name: description: - The name of the host. + - One of O(name) or O(host_id) is required. + type: str + aliases: + - cluster_hostname + cluster: + description: + - The name of the associated (attached) cluster. + - To remove from a cluster, omit and set I(purge=True). + type: str + aliases: + - cluster_name + host_id: + description: + - The unique identifier of the host. Read-only. + - One of O(name) or O(host_id) is required. type: str - required: yes - host_ip: + aliases: + ip_address: description: - - The ip of the host. + - The IP address of the host. type: str - required: no aliases: - - cluster_host_ip + - host_ip rack_id: description: - The rack ID for this host. type: str - required: no - name: + config: description: - - The name of the CM Cluster. + - The host configuration overrides to set. + - To unset a parameter, use V(None) as the value. + type: dict + aliases: + - params + - parameters + host_template: + description: + - The host template (and associated role instances) to apply to the host. type: str - required: no + aliases: + - template + roles: + description: + - Role configuration overrides for the host. + type: list + elements: dict + options: + service: + description: + - The service of the role instance on the host. + type: str + required: yes + aliases: + - service_name + type: + description: + - The role type of the role instance on the host. + type: str + required: yes + aliases: + - role_type + config: + description: + - The host configuration overrides to set. + - To unset a parameter, use V(None) as the value. + type: dict + aliases: + - params + - parameters + role_config_groups: + description: + - Role config groups (and associated role instances) to apply to the host. + type: list + elements: dict + options: + service: + description: + - The service of the role config group (and associated role instance) on the host. + type: str + required: yes + aliases: + - service_name + type: + description: + - The base role type of the role config group (and associated role instance) on the host. + - One of O(type) or O(name) is required. + type: str + aliases: + - role_type + name: + description: + - The name of the role config group (and associated role instance) on the host. + - One of O(type) or O(name) is required. + type: str + tags: + description: + - A set of tags applied to the host. + - To unset a tag, use V(None) as its value. + type: dict + purge: + description: + - Flag for whether the declared role configuration overrides, tags, and associated role instance (via O(host_template) or O(role_config_groups)) should append to existing entries or overwrite, i.e. reset, to only the declared entries. + - To clear all configuration and assignments, set empty dictionaries, e.g. O(config={}), or omit the parameter, e.g. O(role_config_groups), and set O(purge=True). + type: bool + default: False + maintenance: + description: + - Flag for whether the host should be in maintenance mode. + type: bool + aliases: + - maintenance_mode state: description: - State of the host. + - The states V(started), V(stopped), and V(restarted) refer the state of the host's role instances. type: str - default: 'present' + default: present choices: - - 'present' - - 'absent' - - 'attached' - - 'detached' - required: False + - present + - absent + - started + - stopped + - restarted extends_documentation_fragment: - ansible.builtin.action_common_attributes - cloudera.cluster.cm_options - cloudera.cluster.cm_endpoint + - cloudera.cluster.message attributes: check_mode: support: full diff_mode: - support: none + support: full platform: platforms: all +requirements: + - cm-client +seealso: + - module: cloudera.cluster.host_info """ EXAMPLES = r""" @@ -111,86 +208,126 @@ """ RETURN = r""" -cloudera_manager: - description: Details about Cloudera Manager Host - type: dict - contains: - clusterRef: - description: A reference to the enclosing cluster. - type: str - returned: optional - commissionState: - description: Represents the Commission state of an entity. - type: str - returned: optional - distribution: - description: OS distribution details. - type: dict - returned: optional - entity_status: - description: The single value used by the Cloudera Manager UI to represent the status of the entity. - type: str - returned: optional - health_checks: - description: Represents a result from a health test performed by Cloudera Manager for an entity. - type: list - returned: optional - health_summary: - description: The summary status of health check. - type: str - returned: optional - host_id: - description: A unique host identifier. This is not the same as the hostname (FQDN). It is a distinct value that remains the same even if the hostname changes. - type: str - returned: optional - host_url: - description: A URL into the Cloudera Manager web UI for this specific host. - type: str - returned: optional - hostname: - description: The hostname. This field is not mutable after the initial creation. - type: str - returned: optional - ip_address: - description: The host IP address. This field is not mutable after the initial creation. - type: str - returned: optional - last_heartbeat: - description: Time when the host agent sent the last heartbeat. - type: str - returned: optional - maintenance_mode: - description: Maintance mode of Cloudera Manager Service. - type: bool - returned: optional - maintenance_owners: - description: List of Maintance owners for Cloudera Manager Service. - type: list - returned: optional - num_cores: - description: The number of logical CPU cores on this host. - type: number - returned: optional - numPhysicalCores: - description: The number of physical CPU cores on this host. - type: number - returned: optional - rack_id: - description: The rack ID for this host. - type: str - returned: optional - role_refs: - description: The list of roles assigned to this host. - type: list - returned: optional - tags: - description: Tags associated with the host. - type: list - returned: optional - total_phys_mem_bytes: - description: he amount of physical RAM on this host, in bytes. - type: str - returned: optional +host: + description: Details about the host + type: dict + contains: + host_id: + description: + - The unique ID of the host. + - This is not the same as the hostname (FQDN); I(host_id) is a distinct value that remains static across hostname changes. + type: str + returned: always + hostname: + description: The hostname of the host. + type: str + returned: when supported + ip_address: + description: The IP address of the host. + type: str + returned: always + rack_id: + description: The rack ID for this host. + type: str + returned: when supported + last_heartbeat: + description: Time when the host agent sent the last heartbeat. + type: str + returned: when supported + health_summary: + description: The high-level health status of the host. + type: str + returned: always + sample: + - DISABLED + - HISTORY_NOT_AVAILABLE + - NOT_AVAILABLE + - GOOD + - CONCERNING + - BAD + health_checks: + description: Lists all available health checks for the host. + type: list + elements: dict + returned: when supported + contains: + name: + description: Unique name of this health check. + type: str + returned: always + summary: + description: The high-level health status of the health check. + type: str + returned: always + sample: + - DISABLED + - HISTORY_NOT_AVAILABLE + - NOT_AVAILABLE + - GOOD + - CONCERNING + - BAD + explanation: + description: The explanation of this health check. + type: str + returned: when supported + suppressed: + description: + - Whether this health check is suppressed. + - A suppressed health check is not considered when computing the host's overall health. + type: bool + returned: when supported + maintenance_mode: + description: Whether the host is in maintenance mode. + type: bool + returned: when supported + commission_state: + description: Commission state of the host. + type: str + returned: always + maintenance_owners: + description: The list of objects that trigger this host to be in maintenance mode. + type: list + elements: str + returned: when supported + sample: + - CLUSTER + - SERVICE + - ROLE + - HOST + - CONTROL_PLANE + num_cores: + description: The number of logical CPU cores on this host. + type: number + returned: when supported + numPhysicalCores: + description: The number of physical CPU cores on this host. + type: number + returned: when supported + total_phys_mem_bytes: + description: he amount of physical RAM on this host, in bytes. + type: str + returned: when supported + config: + description: Set of host configurations. + type: dict + returned: when supported + distribution: + description: OS distribution details. + type: dict + returned: when supported + tags: + description: The dictionary of tags for the host. + type: dict + returned: when supported + cluster_name: + description: The associated cluster for the host. + type: str + returned: when supported + roles: + description: The list of role instances, i.e. role identifiers, assigned to this host. + type: list + elements: str + returned: when supported """ from cm_client import ( @@ -198,7 +335,6 @@ ApiHostList, ApiHostRef, ApiHostRefList, - ApiRoleConfigGroup, ClustersResourceApi, HostsResourceApi, HostTemplatesResourceApi, @@ -548,11 +684,7 @@ def process(self): self.diff["after"].update(roles=after_ht) # Handle role config group assignment (argspec enforces inclusion of cluster) - # if self.role_config_groups or (not self.host_template and self.purge): if self.role_config_groups: - # if self.role_config_groups is None: - # self.role_config_groups = list() - try: (before_rcg, after_rcg) = reconcile_host_role_config_groups( api_client=self.api_client, @@ -572,11 +704,7 @@ def process(self): self.diff["after"].update(role_config_groups=after_rcg) # Handle role override assignments (argspec enforces inclusion of cluster) - # if self.roles or self.purge: if self.roles: - # if self.roles is None: - # self.roles = list() - try: (before_role, after_role) = reconcile_host_role_configs( api_client=self.api_client, @@ -684,8 +812,6 @@ def main(): choices=[ "present", "absent", - "attached", - "detached", "started", "stopped", "restarted", @@ -695,9 +821,9 @@ def main(): required_one_of=[ ("name", "host_id"), ], - mutually_exclusive=[ - ["host_template", "role_config_groups"], - ], + # mutually_exclusive=[ + # ["host_template", "role_config_groups"], + # ], required_if=[ ("state", "attached", ("cluster",), False), ("state", "started", ("cluster",), False), From 4d01e542b56e61c4cd0fa1116586e3da8f89e81e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:57 -0400 Subject: [PATCH 17/30] Update host_info module to use utility functions and latest logic Signed-off-by: Webster Mudge --- plugins/modules/host_info.py | 361 ++++++++++++++++++++++------------- 1 file changed, 227 insertions(+), 134 deletions(-) diff --git a/plugins/modules/host_info.py b/plugins/modules/host_info.py index 0f479872..771a0b8c 100644 --- a/plugins/modules/host_info.py +++ b/plugins/modules/host_info.py @@ -1,4 +1,7 @@ -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,200 +15,290 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerModule, -) - -from cm_client import HostsResourceApi -from cm_client.rest import ApiException - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" ---- module: host_info -short_description: Gather information about hosts within Cloudera Manager +short_description: Gather information about Cloudera Manager hosts description: - - Gather information about the Cloudera Manager host instance. + - Gather information about the Cloudera Manager host instances. author: - "Ronald Suplina (@rsuplina)" -requirements: - - cm_client + - "Webster Mudge (@wmudge)" options: - cluster_hostname: + cluster: description: - - The name of the host. + - The name of the associated (attached) cluster of the hosts. type: str required: no + aliases: + - cluster_name + name: + description: + - The hostname of the host. + type: str + required: no + aliases: + - cluster_hostname host_id: description: - - The ID of the host. + - The unique identifier of the host. type: str required: no +extends_documentation_fragment: + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint +attributes: + check_mode: + support: full + diff_mode: + support: full + platform: + platforms: all +requirements: + - cm-client +seealso: + - module: cloudera.cluster.host """ EXAMPLES = r""" ---- -- name: Get information about the host with hostname +- name: Get information about the host via hostname + cloudera.cluster.host_info: + host: "example.cloudera.host" + username: "will_jordan" + password: "S&peR4Ec*re" + name: "ecs_node_01.cldr.internal" + +- name: Get information about the host via host id cloudera.cluster.host_info: host: "example.cloudera.host" username: "will_jordan" password: "S&peR4Ec*re" - cluster_hostname: "Ecs_node_01" + host_id: "1a12c6a0-9277-4824-aaa9-38e24a6f5efe" -- name: Get information about the host with host id +- name: Get information about all the hosts registered with Cloudera Manager cloudera.cluster.host_info: host: "example.cloudera.host" username: "will_jordan" password: "S&peR4Ec*re" - cluster_hostname: "Ecs_node_01" -- name: Get information about all the hosts registered by Cloudera Manager +- name: Get information about all the hosts attached to a cluster cloudera.cluster.host_info: host: "example.cloudera.host" username: "will_jordan" password: "S&peR4Ec*re" + cluster: "ExampleCluster" """ RETURN = r""" ---- -cloudera_manager: - description: Details about Cloudera Manager Host - type: list - elements: dict - contains: - hostname: - description: The hostname. This field is not mutable after the initial creation. - type: str - returned: optional - host_id: - description: A unique host identifier. This is not the same as the hostname (FQDN). It is a distinct value that remains the same even if the hostname changes. - type: str - returned: optional - host_url: - description: A URL into the Cloudera Manager web UI for this specific host. - type: str - returned: optional - clusterRef: - description: A reference to the enclosing cluster. - type: str - returned: optional - commissionState: - description: Represents the Commission state of an entity. - type: str - returned: optional - distribution: - description: OS distribution details. - type: dict - returned: optional - entity_status: - description: The single value used by the Cloudera Manager UI to represent the status of the entity. - type: str - returned: optional - health_checks: - description: Represents a result from a health test performed by Cloudera Manager for an entity. - type: list - returned: optional - health_summary: - description: The summary status of health check. - type: str - returned: optional - ip_address: - description: The host IP address. This field is not mutable after the initial creation. - type: str - returned: optional - last_heartbeat: - description: Time when the host agent sent the last heartbeat. - type: str - returned: optional - maintenance_mode: - description: Maintance mode of Cloudera Manager Service. - type: bool - returned: optional - maintenance_owners: - description: List of Maintance owners for Cloudera Manager Service. - type: list - returned: optional - num_cores: - description: The number of logical CPU cores on this host. - type: number - returned: optional - numPhysicalCores: - description: The number of physical CPU cores on this host. - type: number - returned: optional - rack_id: - description: The rack ID for this host. - type: str - returned: optional - role_refs: - description: The list of roles assigned to this host. - type: list - returned: optional - tags: - description: Tags associated with the host. - type: list - returned: optional - total_phys_mem_bytes: - description: he amount of physical RAM on this host, in bytes. - type: str - returned: optional +hosts: + description: Details about Cloudera Manager hosts. + type: list + elements: dict + contains: + host_id: + description: + - The unique ID of the host. + - This is not the same as the hostname (FQDN); I(host_id) is a distinct value that remains static across hostname changes. + type: str + returned: always + hostname: + description: The hostname of the host. + type: str + returned: when supported + ip_address: + description: The IP address of the host. + type: str + returned: always + rack_id: + description: The rack ID for this host. + type: str + returned: when supported + last_heartbeat: + description: Time when the host agent sent the last heartbeat. + type: str + returned: when supported + health_summary: + description: The high-level health status of the host. + type: str + returned: always + sample: + - DISABLED + - HISTORY_NOT_AVAILABLE + - NOT_AVAILABLE + - GOOD + - CONCERNING + - BAD + health_checks: + description: Lists all available health checks for the host. + type: list + elements: dict + returned: when supported + contains: + name: + description: Unique name of this health check. + type: str + returned: always + summary: + description: The high-level health status of the health check. + type: str + returned: always + sample: + - DISABLED + - HISTORY_NOT_AVAILABLE + - NOT_AVAILABLE + - GOOD + - CONCERNING + - BAD + explanation: + description: The explanation of this health check. + type: str + returned: when supported + suppressed: + description: + - Whether this health check is suppressed. + - A suppressed health check is not considered when computing the host's overall health. + type: bool + returned: when supported + maintenance_mode: + description: Whether the host is in maintenance mode. + type: bool + returned: when supported + commission_state: + description: Commission state of the host. + type: str + returned: always + maintenance_owners: + description: The list of objects that trigger this host to be in maintenance mode. + type: list + elements: str + returned: when supported + sample: + - CLUSTER + - SERVICE + - ROLE + - HOST + - CONTROL_PLANE + num_cores: + description: The number of logical CPU cores on this host. + type: number + returned: when supported + numPhysicalCores: + description: The number of physical CPU cores on this host. + type: number + returned: when supported + total_phys_mem_bytes: + description: he amount of physical RAM on this host, in bytes. + type: str + returned: when supported + config: + description: Set of host configurations. + type: dict + returned: when supported + distribution: + description: OS distribution details. + type: dict + returned: when supported + tags: + description: The dictionary of tags for the host. + type: dict + returned: when supported + cluster_name: + description: The associated cluster for the host. + type: str + returned: when supported + roles: + description: The list of role instances, i.e. role identifiers, assigned to this host. + type: list + elements: str + returned: when supported """ +from cm_client import ( + ApiHost, + ClustersResourceApi, + HostsResourceApi, +) +from cm_client.rest import ApiException + -class ClouderaHostInfo(ClouderaManagerModule): +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( + parse_host_result, +) + + +class HostInfo(ClouderaManagerModule): def __init__(self, module): - super(ClouderaHostInfo, self).__init__(module) + super(HostInfo, self).__init__(module) - # Initialize the return values - self.cluster_hostname = self.get_param("cluster_hostname") + # Set the parameters + self.cluster = self.get_param("cluster") + self.name = self.get_param("name") self.host_id = self.get_param("host_id") + + # Initialize the return values + self.output = [] + # Execute the logic self.process() @ClouderaManagerModule.handle_process def process(self): - host_api_instance = HostsResourceApi(self.api_client) - self.host_output = {} - self.changed = False - if self.cluster_hostname or self.host_id: + host_api = HostsResourceApi(self.api_client) + cluster_api = ClustersResourceApi(self.api_client) + + hosts = list[ApiHost]() + + if self.host_id: try: - if self.cluster_hostname: - self.host_output = host_api_instance.read_host( - host_id=self.cluster_hostname - ).to_dict() - else: - self.host_output = host_api_instance.read_host( - host_id=self.host_id - ).to_dict() + hosts.append(host_api.read_host(host_id=self.host_id)) except ApiException as ex: if ex.status != 404: raise ex + elif self.name: + host = next( + (h for h in host_api.read_hosts().items if h.hostname == self.name), + None, + ) + if host is not None: + hosts.append(host) + elif self.cluster: + try: + ClustersResourceApi(self.api_client).read_cluster(self.cluster) + except ApiException as ex: + if ex.status == 404: + self.module.fail_json(msg="Cluster does not exist: " + self.cluster) + else: + raise ex + + hosts = cluster_api.list_hosts( + cluster_name=self.cluster, + ).items else: - self.host_output = host_api_instance.read_hosts().to_dict() + hosts = host_api.read_hosts().items + + for host in hosts: + host.config = host_api.read_host_config(host.host_id) + self.output.append(parse_host_result(host)) def main(): module = ClouderaManagerModule.ansible_module( argument_spec=dict( - cluster_hostname=dict(required=False, type="str"), - host_id=dict(required=False, type="str"), + cluster=dict(aliases=["cluster_name"]), + name=dict(aliases=["cluster_hostname"]), + host_id=dict(), ), supports_check_mode=True, ) - result = ClouderaHostInfo(module) - - changed = result.changed + result = HostInfo(module) output = dict( - changed=changed, - cloudera_manager=result.host_output, + changed=False, + hosts=result.output, ) if result.debug: From 0ca249bf6390cbaef5b5e35e043eadacce0cd3e3 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:58 -0400 Subject: [PATCH 18/30] Rename class for host module Signed-off-by: Webster Mudge --- plugins/modules/host.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index d4a0be37..2f7f78e0 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -369,9 +369,9 @@ ) -class ClusterHost(ClouderaManagerMutableModule): +class Host(ClouderaManagerMutableModule): def __init__(self, module): - super(ClusterHost, self).__init__(module) + super(Host, self).__init__(module) # Set the parameters self.name = self.get_param("name") @@ -838,7 +838,7 @@ def main(): supports_check_mode=True, ) - result = ClusterHost(module) + result = Host(module) output = dict( changed=result.changed, From c311971babb8c39bc580d2a029729954804f5cd8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:58 -0400 Subject: [PATCH 19/30] Remove unused import Signed-off-by: Webster Mudge --- plugins/modules/service_info.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/modules/service_info.py b/plugins/modules/service_info.py index 8821832a..136e061e 100644 --- a/plugins/modules/service_info.py +++ b/plugins/modules/service_info.py @@ -354,7 +354,6 @@ from cm_client import ( ClustersResourceApi, - ServicesResourceApi, ) from cm_client.rest import ApiException @@ -393,8 +392,6 @@ def process(self): else: raise ex - service_api = ServicesResourceApi(self.api_client) - if self.name: try: self.output.append( From 23e391733666c0c7627e7392dbd4b536819c5de4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:55:59 -0400 Subject: [PATCH 20/30] Update tests for host_info to use pytest fixtures Signed-off-by: Webster Mudge --- .../modules/host_info/test_host_info.py | 115 +++++++++++++----- 1 file changed, 84 insertions(+), 31 deletions(-) diff --git a/tests/unit/plugins/modules/host_info/test_host_info.py b/tests/unit/plugins/modules/host_info/test_host_info.py index 23533b79..850e827d 100644 --- a/tests/unit/plugins/modules/host_info/test_host_info.py +++ b/tests/unit/plugins/modules/host_info/test_host_info.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# Copyright 2025 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,11 +17,19 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -import os + import logging import pytest +from cm_client import ( + HostsResourceApi, +) + from ansible_collections.cloudera.cluster.plugins.modules import host_info + +from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( + get_cluster_hosts, +) from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, @@ -30,60 +38,105 @@ LOG = logging.getLogger(__name__) -def test_pytest_hostname_parameter(module_args): +def test_host_info_host_id_invalid(conn, module_args): + module_args( + { + **conn, + "host_id": "BOOM", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_info.main() + + assert not e.value.hosts + + +def test_host_info_name_invalid(conn, module_args): + module_args( + { + **conn, + "name": "BOOM", + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host_info.main() + + assert not e.value.hosts + + +def test_host_info_cluster_invalid(conn, module_args): + module_args( + { + **conn, + "cluster": "BOOM", + } + ) + + with pytest.raises(AnsibleFailJson, match="Cluster does not exist: BOOM"): + host_info.main() + + +def test_host_info_host_id(conn, module_args, cm_api_client): + all_hosts = HostsResourceApi(cm_api_client).read_hosts().items + module_args( { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", - "cluster_hostname": "cloudera.host.example", + **conn, + "host_id": all_hosts[0].host_id, } ) with pytest.raises(AnsibleExitJson) as e: host_info.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) + assert len(e.value.hosts) == 1 + assert e.value.hosts[0]["host_id"] == all_hosts[0].host_id + +def test_host_info_name(conn, module_args, cm_api_client): + all_hosts = HostsResourceApi(cm_api_client).read_hosts().items -def test_pytest_host_id_parameter(module_args): module_args( { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", - "host_id": "cloudera.host.id.example", + **conn, + "name": all_hosts[0].hostname, } ) with pytest.raises(AnsibleExitJson) as e: host_info.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) + assert len(e.value.hosts) == 1 + assert e.value.hosts[0]["host_id"] == all_hosts[0].host_id + +def test_host_info_cluster(conn, module_args, cm_api_client, base_cluster): + cluster_hosts = get_cluster_hosts( + api_client=cm_api_client, + cluster=base_cluster, + ) -def test_pytest_all_hosts(module_args): module_args( { - "username": os.getenv("CM_USERNAME"), - "password": os.getenv("CM_PASSWORD"), - "host": os.getenv("CM_HOST"), - "port": "7180", - "verify_tls": "no", - "debug": "no", + **conn, + "cluster": base_cluster.name, } ) with pytest.raises(AnsibleExitJson) as e: host_info.main() - # LOG.info(str(e.value)) - LOG.info(str(e.value.cloudera_manager)) + assert len(e.value.hosts) == len(cluster_hosts) + + +def test_host_info_all(conn, module_args, cm_api_client): + all_hosts = HostsResourceApi(cm_api_client).read_hosts().items + + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + host_info.main() + + assert len(e.value.hosts) == len(all_hosts) From ff1bacbd0c2238fce8da55ba425fc19ffd3fa3d6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:00 -0400 Subject: [PATCH 21/30] Remove unused imports Signed-off-by: Webster Mudge --- .../unit/plugins/modules/service_info/test_service_info.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/unit/plugins/modules/service_info/test_service_info.py b/tests/unit/plugins/modules/service_info/test_service_info.py index 45a03fa5..207d7491 100644 --- a/tests/unit/plugins/modules/service_info/test_service_info.py +++ b/tests/unit/plugins/modules/service_info/test_service_info.py @@ -19,24 +19,19 @@ __metaclass__ = type import logging -import os import pytest from pathlib import Path from cm_client import ( - ApiConfig, - ApiEntityTag, ApiHost, ApiHostRef, ApiRole, ApiService, - ApiServiceConfig, - ApiServiceState, - ServicesResourceApi, ) from ansible_collections.cloudera.cluster.plugins.modules import service_info + from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( get_cluster_hosts, ) From 1d9d67d2f8ac19eeb55732da3bb369c69d8b9af8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:00 -0400 Subject: [PATCH 22/30] Updated resettable_host() fixture factory to handle cluster attachment Signed-off-by: Webster Mudge --- tests/unit/plugins/modules/host/conftest.py | 56 ++++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/tests/unit/plugins/modules/host/conftest.py b/tests/unit/plugins/modules/host/conftest.py index b0520f1f..c387558c 100644 --- a/tests/unit/plugins/modules/host/conftest.py +++ b/tests/unit/plugins/modules/host/conftest.py @@ -30,6 +30,7 @@ ApiHostRef, ApiHostRefList, ApiHostRef, + ApiHostsToRemoveArgs, ApiRole, ApiRoleList, ApiService, @@ -40,6 +41,7 @@ ) from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + wait_command, wait_commands, TagUpdates, ConfigListUpdates, @@ -244,6 +246,7 @@ def resettable_cluster(cm_api_client, base_cluster): @pytest.fixture() def resettable_host(cm_api_client, request) -> Generator[Callable[[ApiHost], ApiHost]]: host_api = HostsResourceApi(cm_api_client) + cluster_api = ClustersResourceApi(cm_api_client) # Registry of resettable hosts registry = list[ApiHost]() @@ -262,17 +265,17 @@ def _wrapper(host: ApiHost) -> ApiHost: current_hosts_map[host.host_id] = host # Reset each host - for h in registry: - target_host = current_hosts_map.get(h.host_id, None) + for previous_host in registry: + target_host = current_hosts_map.get(previous_host.host_id, None) # If the host was deleted, recreate if target_host is None: - # TODO Handle creation + # TODO Handle host creation pass else: # Tags tag_updates = TagUpdates( - target_host.tags, {t.name: t.value for t in h.tags}, True + target_host.tags, {t.name: t.value for t in previous_host.tags}, True ) if tag_updates.deletions: host_api.delete_tags( @@ -287,11 +290,13 @@ def _wrapper(host: ApiHost) -> ApiHost: ) # Config - if h.config is None: - h.config = ApiConfigList(items=[]) + if previous_host.config is None: + previous_host.config = ApiConfigList(items=[]) config_updates = ConfigListUpdates( - target_host.config, {c.name: c.value for c in h.config.items}, True + target_host.config, + {c.name: c.value for c in previous_host.config.items}, + True, ) host_api.update_host_config( host_id=target_host.host_id, @@ -300,5 +305,42 @@ def _wrapper(host: ApiHost) -> ApiHost: ) # Cluster + if ( + previous_host.cluster_ref is not None + and target_host.cluster_ref is not None + and previous_host.cluster_ref.cluster_name + != target_host.cluster_ref.cluster_name + ) or ( + previous_host.cluster_ref is None + and target_host.cluster_ref is not None + ): + decommission_cmd = host_api.remove_hosts_from_cluster( + body=ApiHostsToRemoveArgs(hosts_to_remove=[target_host.hostname]) + ) + wait_command( + api_client=cm_api_client, + command=decommission_cmd, + ) + + if ( + previous_host.cluster_ref is not None + and target_host.cluster_ref is not None + and previous_host.cluster_ref.cluster_name + != target_host.cluster_ref.cluster_name + ) or ( + previous_host.cluster_ref is not None + and target_host.cluster_ref is None + ): + cluster_api.add_hosts( + cluster_name=previous_host.cluster_ref.cluster_name, + body=ApiHostRefList( + items=[ + ApiHostRef( + host_id=target_host.host_id, + hostname=previous_host.hostname, + ) + ] + ), + ) # Roles From 07430df3c29d81110b250b08ced3966c1aafdecb Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:01 -0400 Subject: [PATCH 23/30] Create tests for host cluster attachment Signed-off-by: Webster Mudge --- .../modules/host/test_host_clusters.py | 154 +++++++++++------- 1 file changed, 96 insertions(+), 58 deletions(-) diff --git a/tests/unit/plugins/modules/host/test_host_clusters.py b/tests/unit/plugins/modules/host/test_host_clusters.py index 85c331f4..311e1551 100644 --- a/tests/unit/plugins/modules/host/test_host_clusters.py +++ b/tests/unit/plugins/modules/host/test_host_clusters.py @@ -20,87 +20,125 @@ import logging import pytest - -from collections.abc import Callable, Generator -from pathlib import Path - -from cm_client import ( - ApiConfig, - ApiConfigList, - ApiEntityTag, - ApiHost, - ApiHostList, - ApiHostRef, - ApiHostRefList, - ApiHostTemplate, - ApiHostTemplateList, - ApiHostRef, - ApiRole, - ApiRoleConfigGroup, - ApiRoleConfigGroupRef, - ApiRoleList, - ApiService, - ClouderaManagerResourceApi, - ClustersResourceApi, - HostsResourceApi, - HostTemplatesResourceApi, - RolesResourceApi, - ServicesResourceApi, -) -from cm_client.rest import ApiException +import random from ansible_collections.cloudera.cluster.plugins.modules import host -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - wait_commands, - TagUpdates, - ConfigListUpdates, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.cluster_utils import ( - get_cluster_hosts, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( - create_role, - provision_service_role, - read_roles, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.host_template_utils import ( - create_host_template_model, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.host_utils import ( - get_host_roles, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( - get_base_role_config_group, -) + from ansible_collections.cloudera.cluster.tests.unit import ( AnsibleExitJson, AnsibleFailJson, - deregister_service, - register_service, ) LOG = logging.getLogger(__name__) -class TestHostAttached: - def test_host_create_invalid_cluster(self, conn, module_args): +class TestHostAttachedCluster: + def test_host_attach_invalid_cluster( + self, conn, module_args, resettable_host, detached_hosts + ): + target_host = resettable_host(random.choice(detached_hosts)) + + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": "BOOM", + } + ) + + with pytest.raises( + AnsibleFailJson, + match="Cluster not found: BOOM", + ): + host.main() + + def test_host_attach_cluster( + self, conn, module_args, base_cluster, resettable_host, detached_hosts + ): + target_host = resettable_host(random.choice(detached_hosts)) + + module_args( + { + **conn, + "name": target_host.hostname, + "cluster": base_cluster.name, + } + ) + + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + assert e.value.host["cluster_name"] == base_cluster.name + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + assert e.value.host["cluster_name"] == base_cluster.name + + +@pytest.mark.skip("Requires set up of two clusters") +class TestHostMigrateClusters: + def test_host_migrate_cluster( + self, conn, module_args, base_cluster, resettable_host, detached_hosts + ): + target_host = resettable_host(random.choice(detached_hosts)) + module_args( { **conn, + "name": target_host.hostname, + "cluster": base_cluster.name, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises(AnsibleExitJson) as e: host.main() + assert e.value.changed == True + assert e.value.host["cluster_name"] == base_cluster.name + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == False + assert e.value.host["cluster_name"] == base_cluster.name + + +class TestHostDetachedCluster: + def test_host_detach(self, conn, module_args, attached_hosts, resettable_host): + target_host = resettable_host(random.choice(attached_hosts)) -class TestHostDetached: - def test_host_create_invalid_cluster(self, conn, module_args): module_args( { **conn, + "name": target_host.hostname, + "purge": True, } ) - with pytest.raises(AnsibleFailJson, match="boom") as e: + with pytest.raises(AnsibleExitJson) as e: + host.main() + + assert e.value.changed == True + assert e.value.host["cluster_name"] == None + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: host.main() + + assert e.value.changed == False + assert e.value.host["cluster_name"] == None + + +@pytest.mark.skip("Requires new host") +class TestHostCreate: + pass + + +@pytest.mark.skip("Requires existing host") +class TestHostDestroy: + pass From e7df8e0ee49e9140be108b93695fb5fcdd866821 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:02 -0400 Subject: [PATCH 24/30] Update error message Signed-off-by: Webster Mudge --- plugins/modules/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index 2f7f78e0..e262e8eb 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -595,7 +595,7 @@ def process(self): except ApiException as ex: if ex.status == 404: self.module.fail_json( - msg=f"Cluster does not exist: {self.cluster}" + msg=f"Cluster not found: {self.cluster}." ) # Handle new cluster membership From 19a5b05527a0d3640de62e776534f25dd1f0e472 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:02 -0400 Subject: [PATCH 25/30] Remove obsolete test Signed-off-by: Webster Mudge --- tests/unit/plugins/modules/host/test_host.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/unit/plugins/modules/host/test_host.py b/tests/unit/plugins/modules/host/test_host.py index 9a271834..f9fa85c8 100644 --- a/tests/unit/plugins/modules/host/test_host.py +++ b/tests/unit/plugins/modules/host/test_host.py @@ -55,21 +55,6 @@ def test_host_missing_required(self, conn, module_args): ) as e: host.main() - def test_host_missing_attached_cluster(self, conn, module_args): - module_args( - { - **conn, - "name": "example", - "state": "attached", - } - ) - - with pytest.raises( - AnsibleFailJson, - match="state is attached but all of the following are missing: cluster", - ) as e: - host.main() - def test_host_missing_host_template_cluster(self, conn, module_args): module_args( { From ae94cea5aa07f59c8f7ada3824b1edc0239af9e9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:03 -0400 Subject: [PATCH 26/30] Add retry to wait_command() utility Signed-off-by: Webster Mudge --- plugins/module_utils/cm_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py index e56f40eb..6e4bb799 100644 --- a/plugins/module_utils/cm_utils.py +++ b/plugins/module_utils/cm_utils.py @@ -74,14 +74,21 @@ def wait_commands( def wait_command( - api_client: ApiClient, command: ApiCommand, polling: int = 120, delay: int = 10 + api_client: ApiClient, + command: ApiCommand, + polling: int = 120, + delay: int = 10, + retry: int = 0, ): poll_count = 0 - while command.active: + retry_count = 0 + + while command.active or retry_count < retry: if poll_count > polling: raise Exception("Command timeout: " + str(command.id)) sleep(delay) poll_count += 1 + retry_count += 1 command = CommandsResourceApi(api_client).read_command(command.id) if not command.success: raise Exception(command.result_message) From 8566ef85e7c4b75a08fdf6a717fdc43a413d5bf2 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:04 -0400 Subject: [PATCH 27/30] Add retry and parcel staging monitoring to host template reconciliation Signed-off-by: Webster Mudge --- plugins/module_utils/host_utils.py | 61 +++++++++++++++++++++++----- plugins/module_utils/parcel_utils.py | 43 +++++++++++++++++++- 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/plugins/module_utils/host_utils.py b/plugins/module_utils/host_utils.py index dee8df0d..7f5c71f0 100644 --- a/plugins/module_utils/host_utils.py +++ b/plugins/module_utils/host_utils.py @@ -16,6 +16,8 @@ A common functions for Cloudera Manager hosts """ +from time import sleep + from cm_client import ( ApiClient, ApiCluster, @@ -55,6 +57,9 @@ read_role, read_roles, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.parcel_utils import ( + wait_parcel_staging, +) from ansible_collections.cloudera.cluster.plugins.module_utils.role_config_group_utils import ( get_base_role_config_group, ) @@ -345,6 +350,12 @@ def reconcile_host_role_config_groups( cluster_rcgs=cluster_rcgs, ) + # Read the parcel states for the cluster until all are at a stable stage + wait_parcel_staging( + api_client=api_client, + cluster=cluster, + ) + # Reconcile the role config groups on the host with the declared role config groups return _reconcile_host_rcgs( api_client=api_client, @@ -532,24 +543,52 @@ def reconcile_host_template_assignments( ) if not check_mode: - # Apply the host template - apply_cmd = host_template_api.apply_host_template( - cluster_name=cluster.name, - host_template_name=host_template.name, - start_roles=False, - body=ApiHostRefList( - items=[ApiHostRef(host_id=host.host_id, hostname=host.hostname)] - ), - ) - wait_command( + # Read the parcel states for the cluster until all are at a stable stage + wait_parcel_staging( api_client=api_client, - command=apply_cmd, + cluster=cluster, ) + # Apply the host template + def _apply(): + apply_cmd = host_template_api.apply_host_template( + cluster_name=cluster.name, + host_template_name=host_template.name, + start_roles=False, + body=ApiHostRefList( + items=[ApiHostRef(host_id=host.host_id, hostname=host.hostname)] + ), + ) + wait_command( + api_client=api_client, + command=apply_cmd, + ) + + retries = 3 + delay = 10 + attempts = 0 + while attempts < retries: + try: + _apply() + break + except ApiException as ae: + attempts += 1 + if ae.status == 400: + sleep(delay) + else: + raise ae + return (diff_before, diff_after) # Else the host has role assignments else: + # Read the parcel states for the cluster until all are at a stable stage + wait_parcel_staging( + api_client=api_client, + cluster=cluster, + ) + + # Reconcile the role assignments of the host template return _reconcile_host_rcgs( api_client=api_client, host=host, diff --git a/plugins/module_utils/parcel_utils.py b/plugins/module_utils/parcel_utils.py index 38a50c5a..a971b712 100644 --- a/plugins/module_utils/parcel_utils.py +++ b/plugins/module_utils/parcel_utils.py @@ -20,13 +20,23 @@ from enum import IntEnum -from cm_client import ApiParcel, ParcelResourceApi +from cm_client import ( + ApiClient, + ApiCluster, + ApiParcel, + ParcelResourceApi, + ParcelsResourceApi, +) from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( normalize_output, ) +class ParcelException(Exception): + pass + + class Parcel(object): STAGE = IntEnum( @@ -140,3 +150,34 @@ def parse_parcel_result(parcel: ApiParcel) -> dict: output = dict(cluster_name=parcel.cluster_ref.cluster_name) output.update(normalize_output(parcel.to_dict(), PARCEL)) return output + + +def wait_parcel_staging( + api_client: ApiClient, cluster: ApiCluster, delay: int = 15, timeout: int = 3600 +) -> None: + parcels_api = ParcelsResourceApi(api_client) + + end_time = time.time() + timeout + + while end_time > time.time(): + # For each cluster parcel, check parcel stage for stable state + parcel_status = [ + parcel + for parcel in parcels_api.read_parcels(cluster_name=cluster.name).items + if parcel.stage + not in [ + "UNAVAILABLE", + "AVAILABLE_REMOTELY", + "DOWNLOADED", + "DISTRIBUTED", + "ACTIVATED", + ] + ] + if not parcel_status: + return + else: + time.sleep(delay) + + raise ParcelException( + f"Failed to reach stable parcel stages for cluster, '{cluster.name}': timeout ({timeout} secs)" + ) From 4b41bf6428aa1d1b040b604c8cc10e7295d0ae6a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:05 -0400 Subject: [PATCH 28/30] Add guard to role type lookups during role model creation Signed-off-by: Webster Mudge --- plugins/module_utils/role_utils.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/plugins/module_utils/role_utils.py b/plugins/module_utils/role_utils.py index eeb6b3d6..0992b027 100644 --- a/plugins/module_utils/role_utils.py +++ b/plugins/module_utils/role_utils.py @@ -233,6 +233,7 @@ def read_cm_roles(api_client: ApiClient) -> ApiRoleList: return ApiRoleList(items=roles) +# TODO Split into CM vs cluster role models def create_role( api_client: ApiClient, role_type: str, @@ -244,18 +245,19 @@ def create_role( role_config_group: str = None, tags: dict = None, ) -> ApiRole: - if ( - role_type.upper() - not in ServicesResourceApi(api_client) - .list_role_types( - cluster_name=cluster_name, - service_name=service_name, - ) - .items - ): - raise InvalidRoleTypeException( - f"Invalid role type '{role_type}' for service '{service_name}'" - ) + if cluster_name and service_name: + if ( + role_type.upper() + not in ServicesResourceApi(api_client) + .list_role_types( + cluster_name=cluster_name, + service_name=service_name, + ) + .items + ): + raise InvalidRoleTypeException( + f"Invalid role type '{role_type}' for service '{service_name}'" + ) # Set up the role type role = ApiRole(type=str(role_type).upper()) From 16deaaa65cc735f813698528252ec99e9c1dc985 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:05 -0400 Subject: [PATCH 29/30] Add AutoTLS toggle and First Run toggle (i.e. force init) Signed-off-by: Webster Mudge --- plugins/modules/cluster.py | 136 ++++++++++++++++++++++++++++++++++--- 1 file changed, 126 insertions(+), 10 deletions(-) diff --git a/plugins/modules/cluster.py b/plugins/modules/cluster.py index a508dba5..1be94492 100644 --- a/plugins/modules/cluster.py +++ b/plugins/modules/cluster.py @@ -17,6 +17,7 @@ from ansible.module_utils.common.text.converters import to_text, to_native from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + wait_command, ClouderaManagerModule, ClusterTemplate, ) @@ -837,6 +838,8 @@ def __init__(self, module): self.contexts = self.get_param("contexts") self.auto_assign = self.get_param("auto_assign") self.control_plane = self.get_param("control_plane") + self.tls = self.get_param("tls") + self.force = self.get_param("force") self.changed = False self.output = {} @@ -912,6 +915,7 @@ def process(self): if not self.module.check_mode: self.cluster_api.auto_assign_roles(cluster_name=self.name) refresh = True + # Create cluster else: # TODO import_cluster_template appears to construct and first run the cluster, which is NOT what present should do @@ -921,6 +925,23 @@ def process(self): else: self.create_cluster_from_parameters() + # Toggle AutoTLS + if self.tls is not None: + if self.tls: + enable_tls_cmd = ( + self.cluster_api.configure_auto_tls_services_command( + cluster_name=self.name + ) + ) + wait_command( + api_client=self.api_client, + command=enable_tls_cmd, + ) + else: + disable_tls_cmd = self.cluster_api.disable_tls( + cluster_name=self.name, + ) + wait_command(api_client=self.api_client, command=disable_tls_cmd) elif self.state == "absent": # Delete cluster refresh = False @@ -951,10 +972,30 @@ def process(self): else: self.create_cluster_from_parameters() + # Toggle AutoTLS + if self.tls is not None: + if self.tls: + enable_tls_cmd = ( + self.cluster_api.configure_auto_tls_services_command( + cluster_name=self.name + ) + ) + wait_command( + api_client=self.api_client, + command=enable_tls_cmd, + ) + else: + disable_tls_cmd = self.cluster_api.disable_tls( + cluster_name=self.name, + ) + wait_command( + api_client=self.api_client, command=disable_tls_cmd + ) + self.changed = True if not self.module.check_mode: # If newly created or created by not yet initialize - if not existing or existing.entity_status == "NONE": + if not existing or existing.entity_status == "NONE" or self.force: first_run = self.cluster_api.first_run(cluster_name=self.name) self.wait_command( first_run, polling=self.timeout, delay=self.delay @@ -980,9 +1021,48 @@ def process(self): self.create_cluster_from_template(template_contents) else: self.create_cluster_from_parameters() + + # Toggle AutoTLS + if self.tls is not None: + if self.tls: + enable_tls_cmd = ( + self.cluster_api.configure_auto_tls_services_command( + cluster_name=self.name + ) + ) + wait_command( + api_client=self.api_client, + command=enable_tls_cmd, + ) + else: + disable_tls_cmd = self.cluster_api.disable_tls( + cluster_name=self.name, + ) + wait_command( + api_client=self.api_client, command=disable_tls_cmd + ) # Stop an existing cluster else: self.changed = True + # Toggle AutoTLS + if self.tls is not None: + if self.tls: + enable_tls_cmd = ( + self.cluster_api.configure_auto_tls_services_command( + cluster_name=self.name + ) + ) + wait_command( + api_client=self.api_client, + command=enable_tls_cmd, + ) + else: + disable_tls_cmd = self.cluster_api.disable_tls( + cluster_name=self.name, + ) + wait_command( + api_client=self.api_client, command=disable_tls_cmd + ) if not self.module.check_mode: stop = self.cluster_api.stop_command(cluster_name=self.name) self.wait_command(stop, polling=self.timeout, delay=self.delay) @@ -1000,8 +1080,33 @@ def process(self): else: self.create_cluster_from_parameters() + # Toggle AutoTLS + if self.tls is not None: + if self.tls: + enable_tls_cmd = ( + self.cluster_api.configure_auto_tls_services_command( + cluster_name=self.name + ) + ) + wait_command( + api_client=self.api_client, + command=enable_tls_cmd, + ) + else: + disable_tls_cmd = self.cluster_api.disable_tls( + cluster_name=self.name, + ) + wait_command( + api_client=self.api_client, command=disable_tls_cmd + ) + self.changed = True if not self.module.check_mode: + if self.force: + first_run = self.cluster_api.first_run(cluster_name=self.name) + self.wait_command( + first_run, polling=self.timeout, delay=self.delay + ) restart = self.cluster_api.restart_command(cluster_name=self.name) self.wait_command(restart, polling=self.timeout, delay=self.delay) @@ -1184,16 +1289,25 @@ def create_cluster_from_parameters(self): # Activate parcels if self.parcels: parcel_api = ParcelResourceApi(self.api_client) - for p, v in self.parcels.items(): - parcel = Parcel( - parcel_api=parcel_api, - product=p, - version=v, - cluster=self.name, - delay=self.delay, - timeout=self.timeout, + try: + for p, v in self.parcels.items(): + parcel = Parcel( + parcel_api=parcel_api, + product=p, + version=v, + cluster=self.name, + delay=self.delay, + timeout=self.timeout, + ) + if self.hosts: + parcel.activate() + else: + parcel.download() + except ApiException as ae: + self.module.fail_json( + msg="Error managing parcel states: " + to_native(ae) ) - parcel.activate() + # Apply host templates for ht, refs in template_map.items(): self.host_template_api.apply_host_template( @@ -1574,6 +1688,8 @@ def main(): contexts=dict(type="list", elements="str", aliases=["data_contexts"]), # Optional enable/disable TLS for the cluster tls=dict(type="bool", aliases=["tls_enabled", "cluster_tls"]), + # Optional force first run services initialization + force=dict(type="bool", aliases=["forced_init"]), # Optional auto-assign roles on cluster (honors existing assignments) auto_assign=dict(type="bool", default=False, aliases=["auto_assign_roles"]), ), From 93dac70d22cd46e5c44726e4d579888586b173d6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 9 May 2025 08:56:06 -0400 Subject: [PATCH 30/30] Add parcel deployment monitoring for cluster membership Signed-off-by: Webster Mudge --- plugins/modules/host.py | 49 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/plugins/modules/host.py b/plugins/modules/host.py index e262e8eb..4f35478d 100644 --- a/plugins/modules/host.py +++ b/plugins/modules/host.py @@ -330,6 +330,7 @@ returned: when supported """ + from cm_client import ( ApiHost, ApiHostList, @@ -338,7 +339,8 @@ ClustersResourceApi, HostsResourceApi, HostTemplatesResourceApi, - RoleConfigGroupsResourceApi, + ParcelResourceApi, + ParcelsResourceApi, RolesResourceApi, ) from cm_client.rest import ApiException @@ -364,6 +366,9 @@ HostMaintenanceStateException, HostException, ) +from ansible_collections.cloudera.cluster.plugins.module_utils.parcel_utils import ( + Parcel, +) from ansible_collections.cloudera.cluster.plugins.module_utils.role_utils import ( parse_role_result, ) @@ -402,7 +407,8 @@ def process(self): cluster_api = ClustersResourceApi(self.api_client) host_api = HostsResourceApi(self.api_client) host_template_api = HostTemplatesResourceApi(self.api_client) - rcg_api = RoleConfigGroupsResourceApi(self.api_client) + parcels_api = ParcelsResourceApi(self.api_client) + parcel_api = ParcelResourceApi(self.api_client) role_api = RolesResourceApi(self.api_client) current = None @@ -607,6 +613,7 @@ def process(self): self.diff["after"].update(cluster=cluster.name) if not self.module.check_mode: + # Add the host to the cluster cluster_api.add_hosts( cluster_name=cluster.name, body=ApiHostRefList( @@ -619,6 +626,23 @@ def process(self): ), ) + parcel_api = ParcelResourceApi(self.api_client) + try: + for parcel in parcels_api.read_parcels( + cluster_name=cluster.name + ).items: + if parcel.stage in ["DOWNLOADED", "DISTRIBUTED"]: + Parcel( + parcel_api=parcel_api, + product=parcel.product, + version=parcel.version, + cluster=cluster.name, + ).activate() + except ApiException as ae: + self.module.fail_json( + msg="Error managing parcel states: " + to_native(ae) + ) + # Handle cluster migration elif current.cluster_ref.cluster_name != cluster.name: self.changed = True @@ -668,14 +692,19 @@ def process(self): msg=f"Host template, '{self.host_template}', does not exist on cluster, '{cluster.name}'" ) - (before_ht, after_ht) = reconcile_host_template_assignments( - api_client=self.api_client, - cluster=cluster, - host=current, - host_template=ht, - purge=self.purge, - check_mode=self.module.check_mode, - ) + try: + (before_ht, after_ht) = reconcile_host_template_assignments( + api_client=self.api_client, + cluster=cluster, + host=current, + host_template=ht, + purge=self.purge, + check_mode=self.module.check_mode, + ) + except ApiException as ex: + self.module.fail_json( + msg=f"Error whil reconciling host template assignments: {to_native(ex)}" + ) if before_ht or after_ht: self.changed = True