From d80ac5c1335685f1e22aec51b8455799192f1a97 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 10:15:13 -0500 Subject: [PATCH 1/9] Update parameter reconcilation and value normalization Signed-off-by: Webster Mudge --- plugins/module_utils/cm_utils.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py index 3fbe3a84..a1667986 100644 --- a/plugins/module_utils/cm_utils.py +++ b/plugins/module_utils/cm_utils.py @@ -104,7 +104,9 @@ def parse_role_config_group_result(role_config_group: ApiRoleConfigGroup) -> dic def normalize_values(add: dict) -> dict: - """Normalize whitespace of parameter values. + """Normalize parameter values. Strings have whitespace trimmed, integers are + converted to strings, and Boolean values are converted their string representation + and lowercased. Args: add (dict): Parameters to review @@ -112,7 +114,18 @@ def normalize_values(add: dict) -> dict: Returns: dict: Normalized parameters """ - return {k: (v.strip() if isinstance(v, str) else v) for k, v in add.items()} + + def _normalize(value): + if isinstance(value, str): + return value.strip() + elif isinstance(value, int): + return str(value) + elif isinstance(value, bool): + return str(value).lower() + else: + return value + + return {k: _normalize(v) for k, v in add.items()} def resolve_parameter_updates( @@ -120,7 +133,8 @@ def resolve_parameter_updates( ) -> dict: """Produce a change set between two parameter dictionaries. - The function will normalize parameter values to remove whitespace. + The function will normalize parameter values to remove whitespace from strings, + convert integers and Booleans to their string representations. Args: current (dict): Existing parameters @@ -131,20 +145,23 @@ def resolve_parameter_updates( dict: A change set of the updates """ updates = {} - diff = recursive_diff(current, incoming) + + diff = recursive_diff(current, normalize_values(incoming)) + if diff is not None: updates = { k: v - for k, v in normalize_values(diff[1]).items() + for k, v in diff[1].items() if k in current or (k not in current and v is not None) } if purge: - # Add the other non-defaults + # Add the remaining non-default values for removal updates = { **updates, **{k: None for k in diff[0].keys() if k not in diff[1]}, } + return updates @@ -384,7 +401,7 @@ def initialize_client(self): """Creates the CM API client""" config = Configuration() - # If provided a CML endpoint URL, use it directly + # If provided a CM endpoint URL, use it directly if self.url: config.host = str(self.url).rstrip(" /") # Otherwise, run discovery on missing parts From 2063642acb96f2191a8f7dd33386a8e02af98f78 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 10:15:15 -0500 Subject: [PATCH 2/9] Create ServiceConfigUpdates class for service-wide configuration management Signed-off-by: Webster Mudge --- plugins/module_utils/service_utils.py | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/service_utils.py b/plugins/module_utils/service_utils.py index f77f3278..c11a2d79 100644 --- a/plugins/module_utils/service_utils.py +++ b/plugins/module_utils/service_utils.py @@ -13,14 +13,19 @@ # limitations under the License. """ -A common functions for Cloudera Manager service management +A common functions for service management """ from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( _parse_output, + resolve_parameter_updates, ) -from cm_client import ApiService +from cm_client import ( + ApiConfig, + ApiService, + ApiServiceConfig, +) SERVICE_OUTPUT = [ "client_config_staleness_status", @@ -44,3 +49,22 @@ def parse_service_result(service: ApiService) -> dict: output = dict(cluster_name=service.cluster_ref.cluster_name) output.update(_parse_output(service.to_dict(), SERVICE_OUTPUT)) return output + + +class ServiceConfigUpdates(object): + def __init__(self, existing: ApiServiceConfig, updates: dict, purge: bool) -> None: + current = {r.name: r.value for r in existing.items} + changeset = resolve_parameter_updates(current, updates, purge) + + self.diff = dict( + before={k: current[k] if k in current else None for k in changeset.keys()}, + after=changeset, + ) + + self.config = ApiServiceConfig( + items=[ApiConfig(name=k, value=v) for k, v in changeset.items()] + ) + + @property + def changed(self) -> bool: + return bool(self.config.items) From 06363d3bcc413406e529a06edb2aae18c7885820 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 10:15:15 -0500 Subject: [PATCH 3/9] Update to use ServiceConfigUpdates class to manage service-wide configuration Signed-off-by: Webster Mudge --- plugins/modules/service_config.py | 67 ++++++++++++------------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/plugins/modules/service_config.py b/plugins/modules/service_config.py index e0a35a19..2f30ca2f 100644 --- a/plugins/modules/service_config.py +++ b/plugins/modules/service_config.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # -*- coding: utf-8 -*- # Copyright 2024 Cloudera, Inc. All Rights Reserved. @@ -14,33 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json - -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerMutableModule, - resolve_parameter_updates, -) - -from cm_client import ( - ApiConfig, - ApiServiceConfig, - ClustersResourceApi, - ServicesResourceApi, -) -from cm_client.rest import ApiException - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" ---- module: service_config -short_description: Manage a service configuration in cluster +short_description: Manage a cluster service configuration description: - - Manage a service configuration (service-wide) in a cluster. + - Manage a configuration (service-wide) for a cluster service. author: - "Webster Mudge (@wmudge)" requirements: @@ -216,6 +195,22 @@ returned: when supported """ +import json + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerMutableModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( + ServiceConfigUpdates, +) + + +from cm_client import ( + ClustersResourceApi, + ServicesResourceApi, +) +from cm_client.rest import ApiException + class ClusterServiceConfig(ClouderaManagerMutableModule): def __init__(self, module): @@ -257,32 +252,22 @@ def process(self): else: raise ex - current = {r.name: r.value for r in existing.items} - incoming = {k: str(v) if v is not None else v for k, v in self.params.items()} - - change_set = resolve_parameter_updates(current, incoming, self.purge) + updates = ServiceConfigUpdates(existing, self.params, self.purge) - if change_set: + if updates.changed: self.changed = True if self.module._diff: - self.diff = dict( - before={ - k: current[k] if k in current else None - for k in change_set.keys() - }, - after=change_set, - ) + self.diff = updates.diff if not self.module.check_mode: - body = ApiServiceConfig( - items=[ApiConfig(name=k, value=v) for k, v in change_set.items()] - ) - self.config = [ p.to_dict() for p in api_instance.update_service_config( - self.cluster, self.service, message=self.message, body=body + self.cluster, + self.service, + message=self.message, + body=updates.config, ).items ] From 563f6f6564795153744929e6f2a4be5dbcab8c6c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 10:15:15 -0500 Subject: [PATCH 4/9] Update documentation Signed-off-by: Webster Mudge --- plugins/modules/service.py | 41 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/plugins/modules/service.py b/plugins/modules/service.py index d80f4300..ff373a7f 100644 --- a/plugins/modules/service.py +++ b/plugins/modules/service.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # -*- coding: utf-8 -*- # Copyright 2024 Cloudera, Inc. All Rights Reserved. @@ -14,29 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( - ClouderaManagerMutableModule, - resolve_tag_updates, -) -from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( - parse_service_result, -) - -from cm_client import ( - ApiEntityTag, - ApiService, - ApiServiceList, - ClustersResourceApi, - ServicesResourceApi, -) -from cm_client.rest import ApiException - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - DOCUMENTATION = r""" module: service short_description: Manage a service in cluster @@ -315,6 +293,23 @@ returned: when supported """ +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerMutableModule, + resolve_tag_updates, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( + parse_service_result, +) + +from cm_client import ( + ApiEntityTag, + ApiService, + ApiServiceList, + ClustersResourceApi, + ServicesResourceApi, +) +from cm_client.rest import ApiException + class ClusterService(ClouderaManagerMutableModule): def __init__(self, module): From 09dea14c67e99d997293c28a5ed2200f6bb70031 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 10:15:15 -0500 Subject: [PATCH 5/9] Add cm_service_config module for service-wide configuration management Signed-off-by: Webster Mudge --- plugins/modules/cm_service_config.py | 282 ++++++++++++++++++ .../test_cm_service_config.py | 175 +++++++++++ 2 files changed, 457 insertions(+) create mode 100644 plugins/modules/cm_service_config.py create mode 100644 tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py diff --git a/plugins/modules/cm_service_config.py b/plugins/modules/cm_service_config.py new file mode 100644 index 00000000..436eba18 --- /dev/null +++ b/plugins/modules/cm_service_config.py @@ -0,0 +1,282 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCUMENTATION = r""" +module: cm_service_config +short_description: Manage the Cloudera Manager service configuration +description: + - Manage a configuration (service-wide) for the Cloudera Manager service. +author: + - "Webster Mudge (@wmudge)" +requirements: + - cm-client +options: + parameters: + description: + - The service-wide configuration to set. + - To unset a parameter, use C(None) as the value. + type: dict + required: yes + aliases: + - params + view: + description: + - The view to materialize. + type: str + default: summary + choices: + - summary + - full +extends_documentation_fragment: + - ansible.builtin.action_common_attributes + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint + - cloudera.cluster.purge + - cloudera.cluster.message +attributes: + check_mode: + support: full + diff_mode: + support: full + platform: + platforms: all +""" + +EXAMPLES = r""" +- name: Update (append) several service-wide parameters + cloudera.cluster.cm_service_config: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + parameters: + a_configuration: "schema://host:port" + another_configuration: 234 + +- name: Reset a service-wide parameter + cloudera.cluster.cm_service_config: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + parameters: + some_conf: None + +- name: Update (purge) service-wide parameters + cloudera.cluster.cm_service_config: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + cluster: example-cluster + service: example-service + parameters: + config_one: ValueOne + config_two: 4567 + purge: yes + +- name: Reset all service-wide parameters + cloudera.cluster.cm_service_config: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + cluster: example-cluster + service: example-service + parameters: {} + purge: yes +""" + +RETURN = r""" +config: + description: Service-wide configuration details for the Cloudera Manager service. + type: list + elements: dict + contains: + name: + description: The canonical name that identifies this configuration parameter. + type: str + returned: always + value: + description: + - The user-defined value. + - When absent, the default value (if any) will be used. + - Can also be absent, when enumerating allowed configs. + type: str + returned: always + required: + description: + - Whether this configuration is required for the service. + - If any required configuration is not set, operations on the service may not work. + - Available using I(view=full). + type: bool + returned: when supported + default: + description: + - The default value. + - Available using I(view=full). + type: str + returned: when supported + display_name: + description: + - A user-friendly name of the parameters, as would have been shown in the web UI. + - Available using I(view=full). + type: str + returned: when supported + description: + description: + - A textual description of the parameter. + - Available using I(view=full). + type: str + returned: when supported + related_name: + description: + - If applicable, contains the related configuration variable used by the source project. + - Available using I(view=full). + type: str + returned: when supported + sensitive: + description: + - Whether this configuration is sensitive, i.e. contains information such as passwords, which might affect how the value of this configuration might be shared by the caller. + type: bool + returned: when supported + validation_state: + description: + - State of the configuration parameter after validation. + - Available using I(view=full). + type: str + returned: when supported + sample: + - OK + - WARNING + - ERROR + validation_message: + description: + - A message explaining the parameter's validation state. + - Available using I(view=full). + type: str + returned: when supported + validation_warnings_suppressed: + description: + - Whether validation warnings associated with this parameter are suppressed. + - In general, suppressed validation warnings are hidden in the Cloudera Manager UI. + - Configurations that do not produce warnings will not contain this field. + - Available using I(view=full). + type: bool + returned: when supported +""" + +import json + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ( + ClouderaManagerMutableModule, +) +from ansible_collections.cloudera.cluster.plugins.module_utils.service_utils import ( + ServiceConfigUpdates, +) + + +from cm_client import ( + MgmtServiceResourceApi, +) +from cm_client.rest import ApiException + + +class ClouderaManagerServiceConfig(ClouderaManagerMutableModule): + def __init__(self, module): + super(ClouderaManagerServiceConfig, self).__init__(module) + + # Set the parameters + self.params = self.get_param("parameters") + self.purge = self.get_param("purge") + self.view = self.get_param("view") + + # Initialize the return value + self.changed = False + self.diff = {} + self.config = [] + + # Execute the logic + self.process() + + @ClouderaManagerMutableModule.handle_process + def process(self): + refresh = True + api_instance = MgmtServiceResourceApi(self.api_client) + + try: + existing = api_instance.read_service_config() + except ApiException as ex: + if ex.status == 404: + self.module.fail_json(msg=json.loads(ex.body)["message"]) + else: + raise ex + + updates = ServiceConfigUpdates(existing, self.params, self.purge) + + if updates.changed: + self.changed = True + + if self.module._diff: + self.diff = updates.diff + + if not self.module.check_mode: + self.config = [ + p.to_dict() + for p in api_instance.update_service_config( + message=self.message, body=updates.config + ).items + ] + + if self.view == "full": + refresh = False + + if refresh: + self.config = [ + p.to_dict() + for p in api_instance.read_service_config(view=self.view).items + ] + + +def main(): + module = ClouderaManagerMutableModule.ansible_module( + argument_spec=dict( + parameters=dict(type="dict", required=True, aliases=["params"]), + purge=dict(type="bool", default=False), + view=dict( + default="summary", + choices=["summary", "full"], + ), + ), + supports_check_mode=True, + ) + + result = ClouderaManagerServiceConfig(module) + + output = dict( + changed=result.changed, + config=result.config, + ) + + if module._diff: + output.update(diff=result.diff) + + if result.debug: + log = result.log_capture.getvalue() + output.update(debug=log, debug_lines=log.split("\n")) + + module.exit_json(**output) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py new file mode 100644 index 00000000..e7f81dbb --- /dev/null +++ b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import os +import pytest + +from ansible_collections.cloudera.cluster.plugins.modules import cm_service_config +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_missing_required(conn, module_args): + module_args(conn) + + with pytest.raises(AnsibleFailJson, match="parameters"): + cm_service_config.main() + + +def test_present_invalid_parameter(conn, module_args): + conn.update( + parameters=dict(example="Example"), + ) + module_args(conn) + + with pytest.raises( + AnsibleFailJson, match="Unknown configuration attribute 'example'" + ): + cm_service_config.main() + + +def test_set_parameters(conn, module_args): + conn.update( + parameters=dict(mgmt_emit_sensitive_data_in_stderr=True), + # _ansible_check_mode=True, + # _ansible_diff=True, + message="test_cm_service_config::test_set_parameters", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == True + assert {c["name"]: c["value"] for c in e.value.config}[ + "mgmt_emit_sensitive_data_in_stderr" + ] == "true" + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == False + assert {c["name"]: c["value"] for c in e.value.config}[ + "mgmt_emit_sensitive_data_in_stderr" + ] == "true" + + +def test_unset_parameters(conn, module_args): + conn.update( + parameters=dict(mgmt_emit_sensitive_data_in_stderr=None), + message="test_cm_service_config::test_unset_parameters", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == True + results = {c["name"]: c["value"] for c in e.value.config} + assert "mgmt_emit_sensitive_data_in_stderr" not in results + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + # Idempotency + assert e.value.changed == False + results = {c["name"]: c["value"] for c in e.value.config} + assert "mgmt_emit_sensitive_data_in_stderr" not in results + + +def test_set_parameters_with_purge(conn, module_args): + conn.update( + parameters=dict(mgmt_emit_sensitive_data_in_stderr=True), + purge=True, + message="test_cm_service_config::test_set_parameters_with_purge", + # _ansible_check_mode=True, + # _ansible_diff=True, + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == True + assert {c["name"]: c["value"] for c in e.value.config}[ + "mgmt_emit_sensitive_data_in_stderr" + ] == "true" + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + # Idempotency + assert e.value.changed == False + assert {c["name"]: c["value"] for c in e.value.config}[ + "mgmt_emit_sensitive_data_in_stderr" + ] == "true" + + +def test_purge_all_parameters(conn, module_args): + conn.update( + parameters=dict(), + purge=True, + message="test_cm_service_config::test_purge_all_parameters", + # _ansible_check_mode=True, + # _ansible_diff=True, + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == True + assert len(e.value.config) == 0 + + # Idempotency + with pytest.raises(AnsibleExitJson) as e: + cm_service_config.main() + + assert e.value.changed == False + assert len(e.value.config) == 0 From 6fa3890f85dc83cdab44b5e603ccc6c5c07810b7 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 14:27:40 -0500 Subject: [PATCH 6/9] Add fixtures for CMS and CMS service-wide configurations Signed-off-by: Webster Mudge --- tests/unit/conftest.py | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3a2a103c..5fb502e1 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -32,11 +32,15 @@ ApiClient, ApiClusterList, ApiCluster, + ApiConfig, ApiHostRef, ApiHostRefList, + ApiService, + ApiServiceConfig, ClustersResourceApi, Configuration, HostsResourceApi, + MgmtServiceResourceApi, ParcelResourceApi, ParcelsResourceApi, ) @@ -243,3 +247,80 @@ def base_cluster(cm_api_client, request): cluster_api.delete_cluster(cluster_name=name) except ApiException as ae: raise Exception(str(ae)) + + +@pytest.fixture(scope="session") +def cms(cm_api_client, request): + """Provisions Cloudera Manager Service.""" + + api = MgmtServiceResourceApi(cm_api_client) + + # Return if the Cloudera Manager Service is already present + try: + yield api.read_service() + return + except ApiException as ae: + if ae.status != 404 or "Cannot find management service." not in str(ae.body): + raise Exception(str(ae)) + + # Provision the Cloudera Manager Service + service = ApiService( + name=request.fixturename, + type="MGMT", + ) + + yield api.setup_cms(body=service) + + api.delete_cms() + + +@pytest.fixture(scope="function") +def cms_service_config(cm_api_client, cms, request): + """Configures service-wide configurations for the Cloudera Manager Service""" + + marker = request.node.get_closest_marker("service_config") + + if marker is None: + raise Exception("No service_config marker found.") + + api = MgmtServiceResourceApi(cm_api_client) + + # Retrieve all of the pre-setup configurations + pre = api.read_service_config() + + # Set the test configurations + # Do so serially, since a failed update due to defaults (see ApiException) will cause remaining + # configuration entries to not run. Long-term solution is to check-and-set, which is + # what the Ansible modules do... + for k, v in marker.args[0].items(): + try: + api.update_service_config( + message=f"{request.node.name}::set", + body=ApiServiceConfig(items=[ApiConfig(name=k, value=v)]), + ) + except ApiException as ae: + if ae.status != 400 or "delete with template" not in str(ae.body): + raise Exception(str(ae)) + + # Yield the Cloudera Manager Service + yield cms + + # Retrieve all of the post-setup configurations + post = api.read_service_config() + + # Reconcile the configurations + pre_set = set([c.name for c in pre.items]) + + reconciled = pre.items.copy() + reconciled.extend( + [ + ApiConfig(name=k.name, value=None) + for k in post.items + if k.name not in pre_set + ] + ) + + api.update_service_config( + message=f"{request.node.name}::reset", + body=ApiServiceConfig(items=reconciled), + ) From 5dcb396a8d414bc836ddfa961dcebd4202d3be3a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 14:28:18 -0500 Subject: [PATCH 7/9] Update tests for CMS service-wide configurations Signed-off-by: Webster Mudge --- .../test_cm_service_config.py | 132 ++++++++---------- 1 file changed, 62 insertions(+), 70 deletions(-) diff --git a/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py index e7f81dbb..570b63b8 100644 --- a/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py +++ b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py @@ -19,7 +19,6 @@ __metaclass__ = type import logging -import os import pytest from ansible_collections.cloudera.cluster.plugins.modules import cm_service_config @@ -31,29 +30,6 @@ LOG = logging.getLogger(__name__) -@pytest.fixture -def conn(): - conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) - - if os.getenv("CM_HOST", None): - conn.update(host=os.getenv("CM_HOST")) - - if os.getenv("CM_PORT", None): - conn.update(port=os.getenv("CM_PORT")) - - if os.getenv("CM_ENDPOINT", None): - conn.update(url=os.getenv("CM_ENDPOINT")) - - if os.getenv("CM_PROXY", None): - conn.update(proxy=os.getenv("CM_PROXY")) - - return { - **conn, - "verify_tls": "no", - "debug": "no", - } - - def test_missing_required(conn, module_args): module_args(conn) @@ -73,93 +49,109 @@ def test_present_invalid_parameter(conn, module_args): cm_service_config.main() -def test_set_parameters(conn, module_args): - conn.update( - parameters=dict(mgmt_emit_sensitive_data_in_stderr=True), - # _ansible_check_mode=True, - # _ansible_diff=True, - message="test_cm_service_config::test_set_parameters", +@pytest.mark.service_config(dict(log_event_retry_frequency=10)) +def test_set_parameters(conn, module_args, cms_service_config): + module_args( + { + **conn, + "parameters": dict(mgmt_emit_sensitive_data_in_stderr=True), + "message": "test_cm_service_config::test_set_parameters", + # _ansible_check_mode=True, + # _ansible_diff=True, + } + ) + + expected = dict( + mgmt_emit_sensitive_data_in_stderr="True", log_event_retry_frequency="10" ) - module_args(conn) with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() assert e.value.changed == True - assert {c["name"]: c["value"] for c in e.value.config}[ - "mgmt_emit_sensitive_data_in_stderr" - ] == "true" + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() # Idempotency with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() assert e.value.changed == False - assert {c["name"]: c["value"] for c in e.value.config}[ - "mgmt_emit_sensitive_data_in_stderr" - ] == "true" + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() -def test_unset_parameters(conn, module_args): - conn.update( - parameters=dict(mgmt_emit_sensitive_data_in_stderr=None), - message="test_cm_service_config::test_unset_parameters", +@pytest.mark.service_config( + dict(mgmt_emit_sensitive_data_in_stderr=True, log_event_retry_frequency=10) +) +def test_unset_parameters(conn, module_args, cms_service_config): + module_args( + { + **conn, + "parameters": dict(mgmt_emit_sensitive_data_in_stderr=None), + "message": "test_cm_service_config::test_unset_parameters", + } ) - module_args(conn) + + expected = dict(log_event_retry_frequency="10") with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() assert e.value.changed == True - results = {c["name"]: c["value"] for c in e.value.config} - assert "mgmt_emit_sensitive_data_in_stderr" not in results + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() + # Idempotency with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() - # Idempotency assert e.value.changed == False - results = {c["name"]: c["value"] for c in e.value.config} - assert "mgmt_emit_sensitive_data_in_stderr" not in results + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() -def test_set_parameters_with_purge(conn, module_args): - conn.update( - parameters=dict(mgmt_emit_sensitive_data_in_stderr=True), - purge=True, - message="test_cm_service_config::test_set_parameters_with_purge", - # _ansible_check_mode=True, - # _ansible_diff=True, +@pytest.mark.service_config( + dict(mgmt_emit_sensitive_data_in_stderr=True, log_event_retry_frequency=10) +) +def test_set_parameters_with_purge(conn, module_args, cms_service_config): + module_args( + { + **conn, + "parameters": dict(mgmt_emit_sensitive_data_in_stderr=True), + "purge": True, + "message": "test_cm_service_config::test_set_parameters_with_purge", + # _ansible_check_mode=True, + # _ansible_diff=True, + } ) - module_args(conn) + + expected = dict(mgmt_emit_sensitive_data_in_stderr="True") with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() assert e.value.changed == True - assert {c["name"]: c["value"] for c in e.value.config}[ - "mgmt_emit_sensitive_data_in_stderr" - ] == "true" + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() + # Idempotency with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() - # Idempotency assert e.value.changed == False - assert {c["name"]: c["value"] for c in e.value.config}[ - "mgmt_emit_sensitive_data_in_stderr" - ] == "true" + assert expected.items() <= {c["name"]: c["value"] for c in e.value.config}.items() -def test_purge_all_parameters(conn, module_args): - conn.update( - parameters=dict(), - purge=True, - message="test_cm_service_config::test_purge_all_parameters", - # _ansible_check_mode=True, - # _ansible_diff=True, +@pytest.mark.service_config( + dict(mgmt_emit_sensitive_data_in_stderr=True, log_event_retry_frequency=10) +) +def test_purge_all_parameters(conn, module_args, cms_service_config): + module_args( + { + **conn, + "parameters": dict(), + "purge": True, + "message": "test_cm_service_config::test_purge_all_parameters", + # _ansible_check_mode=True, + # _ansible_diff=True, + } ) - module_args(conn) with pytest.raises(AnsibleExitJson) as e: cm_service_config.main() From 9e4fa4f7030ca21499f77fd6e911c93b58e07f3f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 16 Dec 2024 14:28:43 -0500 Subject: [PATCH 8/9] Remove deprecated fixtures Signed-off-by: Webster Mudge --- .../service_config/test_service_config.py | 95 ------------------- 1 file changed, 95 deletions(-) diff --git a/tests/unit/plugins/modules/service_config/test_service_config.py b/tests/unit/plugins/modules/service_config/test_service_config.py index 17c446c6..707208a8 100644 --- a/tests/unit/plugins/modules/service_config/test_service_config.py +++ b/tests/unit/plugins/modules/service_config/test_service_config.py @@ -86,101 +86,6 @@ def zk_service_config(cm_api_client, zk_service, request): ) -@pytest.fixture(scope="module") -def target_service(cm_api_client, base_cluster, request): - api = ServicesResourceApi(cm_api_client) - - if os.getenv("CM_SERVICE_NAME", None): - yield api.read_service( - cluster_name=base_cluster.name, service_name=os.getenv("CM_SERVICE_NAME") - ) - else: - cluster_api = ClustersResourceApi(cm_api_client) - name = Path(request.node.name).stem + "_zookeeper" - - service = ApiService( - name=name, - type="ZOOKEEPER", - ) - - api.create_services( - cluster_name=base_cluster.name, body=ApiServiceList(items=[service]) - ) - cluster_api.auto_assign_roles(cluster_name=base_cluster.name) - - # configure = cluster_api.auto_configure(cluster_name=target_cluster.name) - wait_for_command( - cm_api_client, - api.first_run(cluster_name=base_cluster.name, service_name=name), - ) - - yield api.read_service(cluster_name=base_cluster.name, service_name=name) - - api.delete_service(cluster_name=base_cluster.name, service_name=name) - - -@pytest.fixture -def target_service_config(cm_api_client, target_service, request): - marker = request.node.get_closest_marker("prepare") - - if marker is None: - raise Exception("No prepare marker found.") - elif "service_config" not in marker.kwargs: - raise Exception("No 'service_config' parameter found.") - - service_api = ServicesResourceApi(cm_api_client) - - # Retrieve all of the pre-setup configurations - pre = service_api.read_service_config( - cluster_name=target_service.cluster_ref.cluster_name, - service_name=target_service.name, - ) - - # Set the test configurations - # Do so serially, since a failed update due to defaults (see ApiException) will cause remaining - # configuration entries to not run. Long-term solution is to check-and-set, which is - # what the Ansible modules do... - for k, v in marker.kwargs["service_config"].items(): - try: - service_api.update_service_config( - cluster_name=target_service.cluster_ref.cluster_name, - service_name=target_service.name, - message=f"test_service_config::{request.node.name}:set", - body=ApiServiceConfig(items=[ApiConfig(name=k, value=v)]), - ) - except ApiException as ae: - if ae.status != 400 or "delete with template" not in str(ae.body): - raise Exception(str(ae)) - - # Return the targeted service and go run the test - yield target_service - - # Retrieve all of the post-setup configurations - post = service_api.read_service_config( - cluster_name=target_service.cluster_ref.cluster_name, - service_name=target_service.name, - ) - - # Reconcile the configurations - pre_set = set([c.name for c in pre.items]) - - reconciled = pre.items.copy() - reconciled.extend( - [ - ApiConfig(name=k.name, value=None) - for k in post.items - if k.name not in pre_set - ] - ) - - service_api.update_service_config( - cluster_name=target_service.cluster_ref.cluster_name, - service_name=target_service.name, - message=f"test_service_config::{request.node.name}::reset", - body=ApiServiceConfig(items=reconciled), - ) - - def test_missing_required(conn, module_args): module_args(conn) From 7edffb60df46b76bd8dbc5eb199dbd314b2d2617 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Tue, 17 Dec 2024 09:22:51 -0500 Subject: [PATCH 9/9] Fix missing parameter for test_set_parameters Signed-off-by: Webster Mudge --- .../modules/cm_service_config/test_cm_service_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py index 570b63b8..9e208227 100644 --- a/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py +++ b/tests/unit/plugins/modules/cm_service_config/test_cm_service_config.py @@ -49,7 +49,9 @@ def test_present_invalid_parameter(conn, module_args): cm_service_config.main() -@pytest.mark.service_config(dict(log_event_retry_frequency=10)) +@pytest.mark.service_config( + dict(mgmt_emit_sensitive_data_in_stderr=False, log_event_retry_frequency=10) +) def test_set_parameters(conn, module_args, cms_service_config): module_args( {