diff --git a/.gitignore b/.gitignore index 4124e979..0fd389f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,164 @@ -.idea -.vscode -test* -*.pyc -*.bak -.DS_Store -venv +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Keep the Galaxy builds +!cloudera-cluster-*.tar.gz + +# Ignore the test output +tests/output + +# Remove any integration configuration +tests/integration/integration_config.yml + +# Remove the Sphinx build directory +site/_build + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + diff --git a/meta/runtime.yml b/meta/runtime.yml new file mode 100644 index 00000000..e6c9457e --- /dev/null +++ b/meta/runtime.yml @@ -0,0 +1,17 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +requires_ansible: ">=2.10" diff --git a/plugins/doc_fragments/cm_endpoint.py b/plugins/doc_fragments/cm_endpoint.py new file mode 100644 index 00000000..838fad91 --- /dev/null +++ b/plugins/doc_fragments/cm_endpoint.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ModuleDocFragment(object): + DOCUMENTATION = r''' + options: + url: + description: + - The CM API endpoint URL and should include scheme, host, port, and API root path. + - Mutually exclusive with I(host). + type: str + required: False + aliases: + - endpoint + - cm_endpoint_url + ''' diff --git a/plugins/doc_fragments/cm_options.py b/plugins/doc_fragments/cm_options.py new file mode 100644 index 00000000..179ba85b --- /dev/null +++ b/plugins/doc_fragments/cm_options.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ModuleDocFragment(object): + DOCUMENTATION = r''' + options: + host: + description: + - Hostname of the CM API endpoint. + - If set, the C(host) parameter will trigger CM API endpoint discovery, which will follow redirects. + - Mutually exclusive with I(url). + type: str + required: False + aliases: + - hostname + port: + description: + - Port of the CM API endpoint. + - If set, CM API endpoint discovery will connect to the designated port first and will follow redirects. + type: int + required: False + default: 7180 + version: + description: + - API version of the CM API endpoint. + type: str + required: False + default: True + aliases: + - tls + force_tls: + description: + - Flag to force TLS during CM API endpoint discovery. + - If C(False), discovery will first try HTTP and follow any redirects. + type: bool + required: False + default: False + verify_tls: + description: + - Verify the TLS certificates for the CM API endpoint. + type: bool + required: False + default: True + aliases: + - tls + username: + description: + - Username for access to the CM API endpoint. + type: str + required: True + password: + description: + - Password for access to the CM API endpoint. + - This parameter is set to C(no_log). + type: str + required: True + debug: + description: + - Capture the HTTP interaction logs with the CM API endpoint. + type: bool + required: False + default: False + aliases: + - debug_endpoints + agent_header: + description: + - Set the HTTP user agent header when interacting with the CM API endpoint. + type: str + required: False + default: ClouderaFoundry + ''' diff --git a/plugins/doc_fragments/cm_resource.py b/plugins/doc_fragments/cm_resource.py new file mode 100644 index 00000000..2a95b2e4 --- /dev/null +++ b/plugins/doc_fragments/cm_resource.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ModuleDocFragment(object): + DOCUMENTATION = r''' + options: + path: + description: + - Path of the CM API endpoint call. + type: str + required: True + query: + description: + - HTTP query parameters for the CM API endpoint call. + type: dict + aliases: + - query_parameters + - parameters + field: + description: + - Field within the response for result extraction. + - Use I(field) when the returned object has an enclosing field. + type: str + default: 'items' + aliases: + - return_field + ''' diff --git a/plugins/module_utils/cm_utils.py b/plugins/module_utils/cm_utils.py new file mode 100644 index 00000000..4c72e099 --- /dev/null +++ b/plugins/module_utils/cm_utils.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A common Ansible Module for shared functions for Cloudera Manager +""" + +import io +import json +import logging + +from functools import wraps +from urllib3 import disable_warnings +from urllib3.exceptions import InsecureRequestWarning, MaxRetryError, HTTPError +from urllib3.util import Url +from urllib.parse import urljoin + +from ansible.module_utils.basic import AnsibleModule + +from cm_client import ApiClient, Configuration +from cm_client.rest import ApiException, RESTClientObject +from cm_client.apis.cloudera_manager_resource_api import ClouderaManagerResourceApi + + +__credits__ = ["frisch@cloudera.com"] +__maintainer__ = [ + "wmudge@cloudera.com" +] + + +class ClouderaManagerModule(object): + @classmethod + def handle_process(cls, f): + """Wrapper to handle log capture and common HTTP errors""" + @wraps(f) + def _impl(self, *args, **kwargs): + try: + self._initialize_client() + result = f(self, *args, **kwargs) + if self.debug: + self.log_out = self._get_log() + self.log_lines.append(self.log_out.splitlines()) + return result + except ApiException as ae: + body = ae.body.decode('utf-8') + if body != "": + body = json.loads(body) + self.module.fail_json(msg="API error: " + str(ae.reason), + status_code=ae.status, body=body) + except MaxRetryError as maxe: + self.module.fail_json(msg="Request error: " + str(maxe.reason)) + except HTTPError as he: + self.module.fail_json(msg="HTTP request: " + str(he)) + return _impl + + """A base Cloudera Manager (CM) module class""" + def __init__(self, module): + # Set common parameters + self.module = module + self.url = self._get_param('url', None) + self.force_tls = self._get_param('force_tls') + self.host = self._get_param('host') + self.port = self._get_param('port') + self.version = self._get_param('version') + self.username = self._get_param('username') + self.password = self._get_param('password') + self.verify_tls = self._get_param('verify_tls') + self.debug = self._get_param('debug') + self.agent_header = self._get_param('agent_header') + + # Initialize common return values + self.log_out = None + self.log_lines = [] + self.changed = False + + # Configure the core CM API client parameters + config = Configuration() + config.username = self.username + config.password = self.password + config.verify_ssl = self.verify_tls + config.debug = self.debug + + # Configure logging + _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' + if self.debug: + self._setup_logger(logging.DEBUG, _log_format) + self.logger.debug("CM API agent: %s", self.agent_header) + else: + self._setup_logger(logging.ERROR, _log_format) + + if self.verify_tls is False: + disable_warnings(InsecureRequestWarning) + + def _get_param(self, param, default=None): + """Fetches an Ansible input parameter if it exists, else returns optional default or None""" + if self.module is not None: + return self.module.params[param] if param in self.module.params else default + return default + + def _setup_logger(self, log_level, log_format): + """Configures the logging of the HTTP activity""" + self.logger = logging.getLogger('urllib3') + self.logger.setLevel(log_level) + + self.__log_capture = io.StringIO() + handler = logging.StreamHandler(self.__log_capture) + handler.setLevel(log_level) + + formatter = logging.Formatter(log_format) + handler.setFormatter(formatter) + + self.logger.addHandler(handler) + + def _get_log(self): + """Retrieves the contents of the captured log""" + contents = self.__log_capture.getvalue() + self.__log_capture.truncate(0) + return contents + + def _initialize_client(self): + """Configures and creates the API client""" + config = Configuration() + + # If provided a CML endpoint URL, use it directly + if self.url: + config.host = self.url + # Otherwise, run discovery on missing parts + else: + config.host = self._discover_endpoint(config) + + # Create and set the API Client + self.api_client = ApiClient() + + def get_auth_headers(self, config): + """Constructs a Basic Auth header dictionary from the Configuration. + This dictionary can be used directly with the API client's REST client.""" + headers = dict() + auth = config.auth_settings().get('basic') + headers[auth['key']] = auth['value'] + return headers + + def _discover_endpoint(self, config): + """Discovers the scheme and version of a potential Cloudara Manager host""" + # Get the authentication headers and REST client + headers = self.get_auth_headers(config) + rest = RESTClientObject() + + # Resolve redirects to establish HTTP scheme and port + pre_rendered = Url(scheme="https" if self.force_tls else "http", host=self.host, port=self.port) + rendered = rest.pool_manager.request('GET', pre_rendered.url, headers=headers.copy()) + rendered_url = rendered.geturl() + + # Discover API version if not set + if not self.version: + pre_versioned = urljoin(rendered_url, "/api/version") + versioned = rest.pool_manager.request('GET', pre_versioned, headers=headers) + self.version = versioned.data.decode('utf-8') + + # Construct the discovered API endpoint + return urljoin(rendered_url, "/api/" + self.version) + + def set_session_cookie(self): + """Utility to cache the session cookie for intra-module operations.""" + if not self.api_client.last_response: + api_instance = ClouderaManagerResourceApi(self.api_client) + api_instance.get_version() + self.api_client.cookie = self.api_client.last_response.getheader('Set-Cookie') + + def call_api(self, path, method, query=None, field='items', body=None): + """Wrapper to call a CM API endpoint path directly.""" + path_params = [] + header_params = {} + header_params['Accept'] = self.api_client.select_header_accept(['application/json']) + header_params['Content-Type'] = self.api_client.select_header_content_type(['application/json']) + + results =self.api_client.call_api(path, method, path_params, query, + header_params, body, auth_settings=['basic'], + _preload_content=False) + + if 200 >= results[1] <= 299: + data = json.loads(results[0].data.decode('utf-8')) + if field in data: + data = data[field] + return data if type(data) is list else [data] + else: + self.module.fail_json(msg="Error interacting with CM resource", status_code=results[1]) + + + @staticmethod + def ansible_module_discovery(argument_spec={}, required_together=[], **kwargs): + """INTERNAL: Creates the Ansible module argument spec and dependencies for CM API endpoint discovery. + Typically, modules will use the ansible_module method to include direct API endpoint URL support.""" + return AnsibleModule( + argument_spec=dict( + **argument_spec, + host=dict(type='str', aliases=['hostname']), + port=dict(type='int', default=7180), + version=dict(type='str'), + force_tls=dict(type='bool', default=False), + verify_tls=dict(required=False, type='bool', default=True, aliases=['tls']), + username=dict(required=True, type='str'), + password=dict(required=True, type='str', no_log=True), + debug=dict(required=False, type='bool', default=False, aliases=['debug_endpoints']), + agent_header=dict(required=False, type='str', default='ClouderaFoundry') + ), + required_together=required_together + [['username', 'password']], + **kwargs + ) + + @staticmethod + def ansible_module(argument_spec={}, mutually_exclusive=[], required_one_of=[], required_together=[], **kwargs): + """Creates the base Ansible module argument spec and dependencies, including discovery and direct endpoint URL support.""" + return ClouderaManagerModule.ansible_module_discovery( + argument_spec=dict( + **argument_spec, + url=dict(type='str', aliases=['endpoint', 'cm_endpoint_url']), + ), + mutually_exclusive=mutually_exclusive + [['url', 'host']], + required_one_of=required_one_of + [['url', 'host']], + required_together=required_together, + **kwargs + ) diff --git a/plugins/modules/cm_endpoint_info.py b/plugins/modules/cm_endpoint_info.py new file mode 100644 index 00000000..96177f48 --- /dev/null +++ b/plugins/modules/cm_endpoint_info.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClouderaManagerModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: cm_endpoint_info +short_description: Discover the Cloudera Manager API endpoint +description: + - Discover the Cloudera Manager API endpoint. + - The module supports C(check_mode). +author: + - "Webster Mudge (@wmudge)" +requirements: + - cm_client +extends_documentation_fragment: + - cloudera.cluster.cm_options +''' + +EXAMPLES = r''' +--- +# This will first try 'http://example.cloudera.com:7180' and will +# follow any redirects +- name: Gather details using auto-discovery + cloudera.cluster.cm_endpoint_info: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + register: cm_endpoint +''' + +RETURN = r''' +--- +endpoint: + description: The discovered Cloudera Manager API endpoint + type: str + returned: always +''' + +class ClouderaEndpointInfo(ClouderaManagerModule): + def __init__(self, module): + super(ClouderaEndpointInfo, self).__init__(module) + + # Initialize the return values + self.endpoint = "" + + # Execute the logic + self.process() + + @ClouderaManagerModule.handle_process + def process(self): + self.endpoint = self.api_client.host + + +def main(): + module = ClouderaManagerModule.ansible_module_discovery( + supports_check_mode=True + ) + + result = ClouderaEndpointInfo(module) + + output = dict( + changed=False, + endpoint=result.endpoint, + ) + + if result.debug: + output.update( + sdk_out=result.log_out, + sdk_out_lines=result.log_lines + ) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/cm_resource.py b/plugins/modules/cm_resource.py new file mode 100644 index 00000000..9a9319bc --- /dev/null +++ b/plugins/modules/cm_resource.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClouderaManagerModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: cm_resource +short_description: Create, update, and delete resources from the Cloudera Manager API endpoint +description: + - Create, update, and delete resources from ad-hoc Cloudera Manager API endpoint paths, i.e. unimplemented API calls. + - This module only supports the C(POST), C(PUT), and C(DELETE) HTTP methods. + - To retrieve details, i.e. read-only, from ad-hoc/unimplemented API endpoints, use the M(cloudera.cluster.cm_resource_info) module. + - The module supports C(check_mode). +author: + - "Webster Mudge (@wmudge)" +requirements: + - cm_client +options: + method: + description: + - HTTP method for the CM API endpoint path. + type: str + required: True + choices: + - DELETE + - POST + - PUT + body: + description: + - HTTP body for the CM API endpoint call. + type: dict +extends_documentation_fragment: + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint + - cloudera.cluster.cm_resource +''' + +EXAMPLES = r''' +--- +- name: Create a new local Cloudera Manager user + cloudera.cluster.cm_resource: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + path: "/user" + method: "POST" + body: + items: + - name: new_user + password: "Als*$ecU7e" + +- name: Update a Cloudera Manager user + cloudera.cluster.cm_resource: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + path: "/user/existing_user" + method: "PUT" + body: + authRoles: + - name: "ROLE_LIMITED" + +- name: Delete a Cloudera Manager user + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + path: "/user/existing_user" + method: "DELETE" +''' + +RETURN = r''' +--- +resources: + description: + - The results from the Cloudera Manager API endpoint call. + - If the I(field) is found on the response object, its contents will be returned. + type: list + elements: complex + returned: always +''' + +class ClouderaResource(ClouderaManagerModule): + def __init__(self, module): + super(ClouderaResource, self).__init__(module) + + # Set parameters + self.method = self._get_param('method') + self.path = self._get_param('path') + self.query = self._get_param('query', dict()) + self.field = self._get_param('field') + self.body = self._get_param('body') + + # Initialize the return values + self.resources = [] + + # Execute the logic + self.process() + + @ClouderaManagerModule.handle_process + def process(self): + if not self.module.check_mode: + self.resources = self.call_api(self.path, self.method, self.query, + self.field, self.body) + + +def main(): + module = ClouderaManagerModule.ansible_module( + argument_spec=dict( + method=dict(required=True, type='str', choices=['POST', 'PUT', 'DELETE']), + path=dict(required=True, type='str'), + query=dict(required=False, type='dict', aliases=['query_parameters', 'parameters']), + body=dict(required=False, type='dict'), + field=dict(required=False, type='str', default='items', aliases=['return_field']) + ), + supports_check_mode=True + ) + + result = ClouderaResource(module) + + output = dict( + changed=False, + resources=result.resources, + ) + + if result.debug: + output.update( + sdk_out=result.log_out, + sdk_out_lines=result.log_lines + ) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/cm_resource_info.py b/plugins/modules/cm_resource_info.py new file mode 100644 index 00000000..0ffe1b45 --- /dev/null +++ b/plugins/modules/cm_resource_info.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClouderaManagerModule + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: cm_resource_info +short_description: Retrieve resources from the Cloudera Manager API endpoint +description: + - Retrieve resources from ad-hoc Cloudera Manager API endpoint paths, i.e. unimplemented API calls. + - This module only supports the C(GET) HTTP method. + - To interact with ad-hoc/unimplemented API endpoints, use the M(cloudera.cluster.cm_resource) module. + - The module supports C(check_mode). +author: + - "Webster Mudge (@wmudge)" +requirements: + - cm_client +extends_documentation_fragment: + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint + - cloudera.cluster.cm_resource +''' + +EXAMPLES = r''' +--- +- name: Gather details about all Cloudera Manager users + cloudera.cluster.cm_resource_info: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + path: "/users" +''' + +RETURN = r''' +--- +resources: + description: + - The results from the Cloudera Manager API endpoint call. + - If the I(field) is found on the response object, its contents will be returned. + type: list + elements: complex + returned: always +''' + +class ClouderaResourceInfo(ClouderaManagerModule): + def __init__(self, module): + super(ClouderaResourceInfo, self).__init__(module) + + # Set parameters + self.path = self._get_param('path') + self.query = self._get_param('query', dict()) + self.field = self._get_param('field') + + # Initialize the return values + self.resources = [] + + # Execute the logic + self.process() + + @ClouderaManagerModule.handle_process + def process(self): + self.resources = self.call_api(self.path, 'GET', self.query, self.field) + + +def main(): + module = ClouderaManagerModule.ansible_module( + argument_spec=dict( + path=dict(required=True, type='str'), + query=dict(required=False, type='dict', aliases=['query_parameters', 'parameters']), + field=dict(required=False, type='str', default='items', aliases=['return_field']) + ), + supports_check_mode=True + ) + + result = ClouderaResourceInfo(module) + + output = dict( + changed=False, + resources=result.resources, + ) + + if result.debug: + output.update( + sdk_out=result.log_out, + sdk_out_lines=result.log_lines + ) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/cm_version_info.py b/plugins/modules/cm_version_info.py new file mode 100644 index 00000000..ae354236 --- /dev/null +++ b/plugins/modules/cm_version_info.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible_collections.cloudera.cluster.plugins.module_utils.cm_utils import ClouderaManagerModule + +from cm_client import ClouderaManagerResourceApi + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = r''' +--- +module: cm_version_info +short_description: Gather information about Cloudera Manager +description: + - Gather information about the Cloudera Manager instance. + - The module supports C(check_mode). +author: + - "Webster Mudge (@wmudge)" +requirements: + - cm_client +extends_documentation_fragment: + - cloudera.cluster.cm_options + - cloudera.cluster.cm_endpoint +''' + +EXAMPLES = r''' +--- +- name: Gather details using an endpoint URL + cloudera.cluster.cm_version: + url: "https://example.cloudera.com:7183/api/v49" + username: "jane_smith" + password: "S&peR4Ec*re" + register: cm_output + +# This will first try 'http://example.cloudera.com:7180' and will +# follow any redirects +- name: Gather details using auto-discovery + cloudera.cluster.cm_version: + host: example.cloudera.com + username: "jane_smith" + password: "S&peR4Ec*re" + register: cm_discovery +''' + +RETURN = r''' +--- +version: + description: Details for the Cloudera Manager instance + type: dict + contains: + version: + description: The CM version. + type: str + returned: optional + snapshot: + description: Whether this build is a development snapshot. + type: bool + returned: optional + build_user: + description: The user performing the build. + type: str + returned: optional + build_timestamp: + description: Build timestamp. + type: str + returned: optional + git_hash: + description: Source control management hash. + type: str + returned: optional +''' + +class ClouderaManagerVersionInfo(ClouderaManagerModule): + def __init__(self, module): + super(ClouderaManagerVersionInfo, self).__init__(module) + + # Initialize the return values + self.cm = dict() + + # Execute the logic + self.process() + + @ClouderaManagerModule.handle_process + def process(self): + api_instance = ClouderaManagerResourceApi(self.api_client) + self.cm=api_instance.get_version().to_dict() + + +def main(): + module = ClouderaManagerModule.ansible_module( + supports_check_mode=True + ) + + result = ClouderaManagerVersionInfo(module) + + output = dict( + changed=False, + cm=result.cm, + ) + + if result.debug: + output.update( + sdk_out=result.log_out, + sdk_out_lines=result.log_lines + ) + + module.exit_json(**output) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..f6cbd92d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,17 @@ +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[pytest] +filterwarnings = + ignore::DeprecationWarning \ No newline at end of file diff --git a/roles/cloudera_manager/config/filter_plugins/filters.py b/roles/cloudera_manager/api_client/handlers/main.yml similarity index 59% rename from roles/cloudera_manager/config/filter_plugins/filters.py rename to roles/cloudera_manager/api_client/handlers/main.yml index 688e7065..9a9530f3 100644 --- a/roles/cloudera_manager/config/filter_plugins/filters.py +++ b/roles/cloudera_manager/api_client/handlers/main.yml @@ -1,4 +1,3 @@ -#!/usr/bin/python # Copyright 2021 Cloudera, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,17 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- -class FilterModule(object): - - def filters(self): - return { - 'filter_null_configs': self.filter_null_configs - } - - def filter_null_configs(self, configs, existing_configs): - filtered_configs = dict(configs) - for item, value in configs.items(): - if item not in existing_configs and not value: - del filtered_configs[item] - return filtered_configs +- name: restart cloudera management service + cm_api: + endpoint: /cm/service/commands/restart + method: POST + timeout: "{{ cluster_restart_timeout | default(3000) }}" \ No newline at end of file diff --git a/roles/cloudera_manager/api_hosts/tasks/main.yml b/roles/cloudera_manager/api_hosts/tasks/main.yml index 28e80b8a..bc20cc97 100644 --- a/roles/cloudera_manager/api_hosts/tasks/main.yml +++ b/roles/cloudera_manager/api_hosts/tasks/main.yml @@ -15,6 +15,7 @@ --- - name: Get the host identifiers and names from Cloudera Manager + delegate_to: "{{ groups.cloudera_manager[0] if 'cloudera_manager' in groups else 'localhost' }}" cloudera.cluster.cm_api: endpoint: /hosts method: GET diff --git a/roles/cloudera_manager/autotls/files/cert.py_patch b/roles/cloudera_manager/autotls/files/cert.py_patch new file mode 100644 index 00000000..e3a12116 --- /dev/null +++ b/roles/cloudera_manager/autotls/files/cert.py_patch @@ -0,0 +1,11 @@ +--- cert.py 2020-12-02 00:54:05.000000000 +0100 ++++ cert.py_2 2021-02-18 09:09:38.095192730 +0100 +@@ -1949,7 +1949,7 @@ + LOG.info("Could not find JKS truststore at location: %s. Converting " + "PEM truststore to JKS." % cluster_ca_jks) + generate_truststore(self.cfg.keytool, cluster_ca_jks, truststore_password, +- cluster_ca_pem) ++ cluster_ca_pem, self.cfg.keystore_type) + + global_ca_pem = self.trust_files[GLOBAL_TLS_SET][PEM_TLS_TYPE] + copied_cluster_to_global = False diff --git a/roles/cloudera_manager/autotls/tasks/main.yml b/roles/cloudera_manager/autotls/tasks/main.yml index dc3308f0..736f6eb7 100644 --- a/roles/cloudera_manager/autotls/tasks/main.yml +++ b/roles/cloudera_manager/autotls/tasks/main.yml @@ -23,23 +23,48 @@ msg: This playbook requires Cloudera Manager 7.1+ when: response.json.version is version('7.1', '<') +- name: Patch Cloudera Manager older than 7.3 + include_tasks: + file: patch_old_cm + when: response.json.version is version('7.3.0', '<') + +- name: Check if password or key is used to connect to machines + set_fact: + use_password: "{{ true if node_password is defined and node_password|length > 0 else false }}" + +- name: DEBUG Auto-TLS using password + debug: + msg: "{{ lookup('template', 'auto-tls.json') }}" + when: use_password and debug | default(false) + - name: Enable Auto-TLS - cloudera.cluster.cm_api: - endpoint: /cm/commands/generateCmca + cm_api: + endpoint: "/cm/commands/generateCmca" method: POST - body: "{{ lookup('template', 'request.j2', convert_data=False) }}" + body: "{{ lookup('template', 'auto-tls.json') }}" + timeout: 360 + ignore_errors: true + when: use_password -- name: Restart Cloudera Manager Server - service: - name: cloudera-scm-server - state: restarted - notify: - - wait cloudera-scm-server +- name: Set node_key on one line + set_fact: + node_key_one_line: "{{ lookup('file', '~/node_key' ) | replace('\n', '\\n') | replace('\"', '\\\"' ) }}" + when: not use_password -- name: Wait for Cloudera Manager Server to come back up - meta: flush_handlers +- name: DEBUG Auto-TLS using key + debug: + msg: "{{ lookup('template', 'auto-tls-key.json') }}" + when: not use_password -- name: Restart Cloudera Management Service - cloudera.cluster.cm_api: - endpoint: /cm/service/commands/restart +- name: Enable Auto-TLS + cm_api: + endpoint: "/cm/commands/generateCmca" method: POST + body: "{{ lookup('template', 'auto-tls-key.json') }}" + ignore_errors: true + when: not use_password + notify: + - restart cloudera-scm-server + - restart cloudera management service + - restart cloudera-scm-agent + diff --git a/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml b/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml new file mode 100644 index 00000000..62182247 --- /dev/null +++ b/roles/cloudera_manager/autotls/tasks/patch_old_cm.yml @@ -0,0 +1,16 @@ +--- +- name: Copy patch to machines + copy: + src: "{{ role_path}}/files/cert.py_patch" + dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py_patch + owner: cloudera-scm + group: cloudera-scm + mode: '0644' + +- name: Backup cert.py + shell: cp /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py.backup + +- name: Fix cert.py + ansible.posix.patch: + src: "{{ role_path}}/patch/cert.py_patch" + dest: /opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/tools/cert.py \ No newline at end of file diff --git a/roles/cloudera_manager/autotls/templates/auto-tls-key.json b/roles/cloudera_manager/autotls/templates/auto-tls-key.json new file mode 100644 index 00000000..356adb7d --- /dev/null +++ b/roles/cloudera_manager/autotls/templates/auto-tls-key.json @@ -0,0 +1,9 @@ +{ + "customCA" : false, + "configureAllServices" : "true", + "sshPort" : 22, + {% if freeipa_activated %}"trustedCaCerts" : "/etc/ipa/ca.crt",{% endif %} + "userName" : "root", + "privateKey": "{{ node_key_one_line }}" +} + \ No newline at end of file diff --git a/roles/cloudera_manager/autotls/templates/auto-tls.json b/roles/cloudera_manager/autotls/templates/auto-tls.json new file mode 100644 index 00000000..467a0ce2 --- /dev/null +++ b/roles/cloudera_manager/autotls/templates/auto-tls.json @@ -0,0 +1,9 @@ +{ + "customCA" : false, + "configureAllServices" : "true", + "sshPort" : 22, + {% if freeipa_activated %}"trustedCaCerts" : "/etc/ipa/ca.crt",{% endif %} + "userName" : "root", + "password": "{{ node_password }}" +} + \ No newline at end of file diff --git a/roles/cloudera_manager/autotls/templates/request.j2 b/roles/cloudera_manager/autotls/templates/request.j2 deleted file mode 100644 index 448824bc..00000000 --- a/roles/cloudera_manager/autotls/templates/request.j2 +++ /dev/null @@ -1,18 +0,0 @@ -{ - "location": "", - "customCA": false, - "interpretAsFilenames": true, - "cmHostCert": "", - "cmHostKey": "", - "caCert": "", - "keystorePasswd": "", - "truststorePasswd": "", - "trustedCaCerts": "", - "hostCerts": [], - "configureAllServices": true, - "sshPort": 22, - "userName": "{{ host_ssh_username }}", - "password": "{{ host_ssh_password }}", - "privateKey": "", - "passphrase": "" -} diff --git a/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json new file mode 100644 index 00000000..74056c7c --- /dev/null +++ b/roles/cloudera_manager/cms_tls/files/cms_keystore_tls.json @@ -0,0 +1,16 @@ +{ + "items": [ + { + "name": "ssl_server_keystore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_enabled", + "value": "true" + } + ] + } \ No newline at end of file diff --git a/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json new file mode 100644 index 00000000..3e139999 --- /dev/null +++ b/roles/cloudera_manager/cms_tls/files/cms_navigator_keystore_tls.json @@ -0,0 +1,28 @@ +{ + "items": [ + { + "name": "navigator_truststore_file", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "navigator_truststore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_keypassword", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_enabled", + "value": "true" + } + ] + } \ No newline at end of file diff --git a/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json new file mode 100644 index 00000000..249ed5be --- /dev/null +++ b/roles/cloudera_manager/cms_tls/files/cms_navigator_metaserver_keystore_tls.json @@ -0,0 +1,20 @@ +{ + "items": [ + { + "name": "ssl_server_keystore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_keypassword", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_enabled", + "value": "true" + } + ] + } \ No newline at end of file diff --git a/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json b/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json new file mode 100644 index 00000000..261ae793 --- /dev/null +++ b/roles/cloudera_manager/cms_tls/files/cms_truststore_tls.json @@ -0,0 +1,12 @@ +{ + "items": [ + { + "name": "ssl_client_truststore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_client_truststore_password", + "value": "{{CM_AUTO_TLS}}" + } + ] + } \ No newline at end of file diff --git a/roles/cloudera_manager/cms_tls/meta/main.yml b/roles/cloudera_manager/cms_tls/meta/main.yml new file mode 100644 index 00000000..c69299ca --- /dev/null +++ b/roles/cloudera_manager/cms_tls/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: cloudera.cluster.cloudera_manager.api_client \ No newline at end of file diff --git a/roles/cloudera_manager/cms_tls/tasks/main.yml b/roles/cloudera_manager/cms_tls/tasks/main.yml new file mode 100644 index 00000000..b3949cfc --- /dev/null +++ b/roles/cloudera_manager/cms_tls/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Setup TLS for Activity Monitor + cm_api: + method: PUT + endpoint: /cm/service/roleConfigGroups/mgmt-ACTIVITYMONITOR-BASE/config + body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" + +- name: Setup TLS for Host Monitor + cm_api: + method: PUT + endpoint: /cm/service/roleConfigGroups/mgmt-HOSTMONITOR-BASE/config + body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" + +- name: Setup TLS for Service Monitor + cm_api: + method: PUT + endpoint: /cm/service/roleConfigGroups/mgmt-SERVICEMONITOR-BASE/config + body: "{{ lookup('file', 'cms_keystore_tls.json', convert_data=False) }}" + +- name: Setup TLS for Navigator + cm_api: + method: PUT + endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATOR-BASE/config + body: "{{ lookup('file', 'cms_navigator_keystore_tls.json', convert_data=False) }}" + when: cloudera_manager_version is version('7.0.0','<') + +- name: Setup TLS for Navigator Meta Server + cm_api: + method: PUT + endpoint: /cm/service/roleConfigGroups/mgmt-NAVIGATORMETASERVER-BASE/config + body: "{{ lookup('file', 'cms_navigator_metaserver_keystore_tls.json', convert_data=False) }}" + when: cloudera_manager_version is version('7.0.0','<') + +- name: Setup TLS for CMS + cm_api: + method: PUT + endpoint: /cm/service/config + body: "{{ lookup('file', 'cms_truststore_tls.json', convert_data=False) }}" + notify: + - restart cloudera management service diff --git a/roles/cloudera_manager/config/meta/main.yml b/roles/cloudera_manager/config/meta/main.yml index 65a7d136..0ae36e31 100644 --- a/roles/cloudera_manager/config/meta/main.yml +++ b/roles/cloudera_manager/config/meta/main.yml @@ -14,6 +14,6 @@ --- dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/config/tasks/main.yml b/roles/cloudera_manager/config/tasks/main.yml index d306d718..4a1fdf63 100644 --- a/roles/cloudera_manager/config/tasks/main.yml +++ b/roles/cloudera_manager/config/tasks/main.yml @@ -15,6 +15,7 @@ --- - name: Get existing configs + delegate_to: "{{ groups.cloudera_manager[0] if 'cloudera_manager' in groups else 'localhost' }}" cloudera.cluster.cm_api: endpoint: "{{ api_config_endpoint }}" register: response @@ -31,6 +32,7 @@ when: message is defined and "message" not in api_config_endpoint - name: Update configuration (via Cloudera Manager API) + delegate_to: "{{ groups.cloudera_manager[0] if 'cloudera_manager' in groups else 'localhost' }}" cloudera.cluster.cm_api: endpoint: "{{ api_config_endpoint }}" body: "{{ lookup('template', 'config.j2', convert_data=False) }}" diff --git a/roles/cloudera_manager/external_auth/defaults/main.yml b/roles/cloudera_manager/external_auth/defaults/main.yml index 5df31ba9..877b72aa 100644 --- a/roles/cloudera_manager/external_auth/defaults/main.yml +++ b/roles/cloudera_manager/external_auth/defaults/main.yml @@ -13,7 +13,31 @@ # limitations under the License. --- - cloudera_manager_external_auth: + provider: "{{ 'FreeIPA' if freeipa_activated == true else omit }}" external_first: no external_only: no + external_set: "{{ 'yes' if freeipa_activated == true else 'no' }}" + role_mappings: "{{ default_free_ipa_role_mappings if freeipa_activated == true else omit }}" + +default_free_ipa_role_mappings: + - group: admins + roles: [ ROLE_ADMIN ] + - group: auditors + roles: [ ROLE_AUDITOR ] + - group: users + roles: [ ROLE_USER ] + +auth_providers: + FreeIPA: + type: LDAP + ldap_url: "{{ ipa_ldap_url }}" + ldap_base_dn: + ldap_bind_user_dn: "{{ ipa_ldap_user_bind_dn }}" + ldap_bind_password: "{{ ipa_ldap_user_bind_password }}" + ldap_search_base: + user: "{{ ipa_ldap_user_search_base }}" + group: "{{ ipa_ldap_group_search_base }}" + ldap_search_filter: + user: "{{ ipa_ldap_user_search_filter }}" + group: "{{ ipa_ldap_user_group_filter }}" \ No newline at end of file diff --git a/roles/cloudera_manager/external_auth/meta/main.yml b/roles/cloudera_manager/external_auth/meta/main.yml index 3a5a0054..ca9fad9c 100644 --- a/roles/cloudera_manager/external_auth/meta/main.yml +++ b/roles/cloudera_manager/external_auth/meta/main.yml @@ -16,3 +16,4 @@ dependencies: - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.infrastructure.krb5_common diff --git a/roles/cloudera_manager/kerberos/tasks/main.yml b/roles/cloudera_manager/kerberos/tasks/main.yml index a15ff7b7..71038722 100644 --- a/roles/cloudera_manager/kerberos/tasks/main.yml +++ b/roles/cloudera_manager/kerberos/tasks/main.yml @@ -26,6 +26,9 @@ endpoint: /cm/commands/importAdminCredentials?username={{ krb5_kdc_admin_user | urlencode }}&password={{ krb5_kdc_admin_password | urlencode }} method: POST register: result + failed_when: + - result is failed + - "'already exists' not in result.content" until: result is not failed retries: 3 delay: 10 diff --git a/roles/cloudera_manager/repo/defaults/main.yml b/roles/cloudera_manager/repo/defaults/main.yml index b0345f7b..35a82308 100644 --- a/roles/cloudera_manager/repo/defaults/main.yml +++ b/roles/cloudera_manager/repo/defaults/main.yml @@ -14,6 +14,10 @@ --- cloudera_archive_base_url: https://archive.cloudera.com -cloudera_manager_version: 7.4.4 +cloudera_manager_version: 7.6.1 +cloudera_manager_distro_name: "{{ ansible_os_family | lower }}" +cloudera_manager_distro_version: "{{ ansible_distribution_major_version }}" -install_repo_on_host: yes \ No newline at end of file +install_repo_on_host: yes + +set_custom_repo_as_archive_base_url: "{{ use_custom_repo_as_archive_base_url | default(True) }}" \ No newline at end of file diff --git a/roles/cloudera_manager/repo/tasks/main-RedHat.yml b/roles/cloudera_manager/repo/tasks/main-RedHat.yml index 1d67551f..c7947db9 100644 --- a/roles/cloudera_manager/repo/tasks/main-RedHat.yml +++ b/roles/cloudera_manager/repo/tasks/main-RedHat.yml @@ -28,4 +28,4 @@ - name: yum-clean-metadata command: yum clean metadata args: - warn: no \ No newline at end of file + warn: no diff --git a/roles/cloudera_manager/repo/tasks/main.yml b/roles/cloudera_manager/repo/tasks/main.yml index 578127e1..c461c175 100644 --- a/roles/cloudera_manager/repo/tasks/main.yml +++ b/roles/cloudera_manager/repo/tasks/main.yml @@ -18,6 +18,13 @@ include_vars: file: "{{ ansible_os_family }}.yml" +- name: Use Custom Repo as Archive Base if using Custom Repo + when: + - set_custom_repo_as_archive_base_url | bool + - '"custom_repo" in groups' + ansible.builtin.set_fact: + cloudera_archive_base_url: "http://{{ groups['custom_repo'] | first }}" + - name: Correct repo URL for Redhat with cm5 ansible.builtin.set_fact: __cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/p/cm{{ __cloudera_manager_major_version }}/redhat/{{ ansible_distribution_major_version }}/x86_64/cm/{{ cloudera_manager_version }}" @@ -45,4 +52,4 @@ - name: Install Cloudera Manager repository when: install_repo_on_host include_tasks: - file: "main-{{ ansible_os_family }}.yml" + file: "main-{{ ansible_os_family }}.yml" \ No newline at end of file diff --git a/roles/cloudera_manager/repo/vars/RedHat.yml b/roles/cloudera_manager/repo/vars/RedHat.yml index d16a422f..b7f5f8e3 100644 --- a/roles/cloudera_manager/repo/vars/RedHat.yml +++ b/roles/cloudera_manager/repo/vars/RedHat.yml @@ -13,12 +13,11 @@ # limitations under the License. --- -__cloudera_manager_distro_name: "{{ ansible_os_family | lower }}{{ ansible_distribution_major_version }}" __cloudera_manager_major_version: "{{ cloudera_manager_version.split('.')[0] }}" __cloudera_manager_cm5_path: "{{ ansible_os_family | lower }}/{{ ansible_distribution_major_version }}/x86_64/cm/{{ cloudera_manager_version }}" -__cloudera_manager_cm6_path: "{{ cloudera_manager_version }}/{{ __cloudera_manager_distro_name }}/yum" +__cloudera_manager_cm6_path: "{{ cloudera_manager_version }}/{{ cloudera_manager_distro_name }}{{ cloudera_manager_distro_version }}/yum" -__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/{{ __cloudera_manager_distro_name }}/yum" +__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/{{ cloudera_manager_distro_name }}{{ cloudera_manager_distro_version }}/yum" __cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/p/cm{{ __cloudera_manager_major_version }}/{{ (__cloudera_manager_major_version == '5' ) | ternary(__cloudera_manager_cm5_path, __cloudera_manager_cm6_path) }}" __cloudera_manager_repo_key_filename: "RPM-GPG-KEY-cloudera" diff --git a/roles/cloudera_manager/services_info/defaults/main.yml b/roles/cloudera_manager/services_info/defaults/main.yml new file mode 100644 index 00000000..66abcb01 --- /dev/null +++ b/roles/cloudera_manager/services_info/defaults/main.yml @@ -0,0 +1,6 @@ +cluster_name: Default +ranger_user: "{{ ranger_rangeradmin_user | default('admin') }}" +ranger_password: "{{ ranger_rangeradmin_user_password | default(cloudera_manager_admin_password) }}" +solr_admin_password: "{{ solr_solradmin_user_password | default(cloudera_manager_admin_password) }}" + +wxm_api_port: 12022 \ No newline at end of file diff --git a/roles/operations/restart_agents/tasks/main.yml b/roles/cloudera_manager/services_info/meta/main.yml similarity index 86% rename from roles/operations/restart_agents/tasks/main.yml rename to roles/cloudera_manager/services_info/meta/main.yml index d44056b5..3a5a0054 100644 --- a/roles/operations/restart_agents/tasks/main.yml +++ b/roles/cloudera_manager/services_info/meta/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- -- name: restart cloudera-scm-agent - service: - name: cloudera-scm-agent - state: restarted \ No newline at end of file + +dependencies: + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/services_info/tasks/main.yml b/roles/cloudera_manager/services_info/tasks/main.yml new file mode 100644 index 00000000..47159804 --- /dev/null +++ b/roles/cloudera_manager/services_info/tasks/main.yml @@ -0,0 +1,323 @@ +--- + +- name: Get All services from CM + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services" + register: cloudera_manager_all_services + no_log: yes # overly verbose + +- name: Get All Mgmt Roles from CM + cloudera.cluster.cm_api: + endpoint: "/cm/service/roles" + register: cloudera_manager_mgmt_roles + no_log: yes # overly verbose + +- name: Get CM Hosts info + cloudera.cluster.cm_api: + endpoint: "/hosts" + register: hosts_details + no_log: yes # overly verbose + +- name: Get CM deployment of services + cloudera.cluster.cm_api: + endpoint: "/cm/deployment" + register: cm_deployment_services + no_log: yes # overly verbose + +- name: Get cluster parcel details + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster_name | urlencode() }}/parcels + register: parcels_response + +- name: Extract active parcels + set_fact: + active_parcels: > + {{ parcels_response.json | + json_query('items[?stage==`ACTIVATED`].{product: product, version: version}') }} + +- name: Extract version of active runtime parcel + set_fact: + runtime_parcel_version: > + {{ active_parcels | + json_query('[?product==`CDH`].version') | first }} + +- name: Use KeyTrustee-based KMS services for CDH 5.x or 6.x + set_fact: + kms_service_type: KEYTRUSTEE + when: + - runtime_parcel_version is version('5.0.0', '>=') + - runtime_parcel_version is version('7.0.0', '<') + +- name: Use Ranger-based KMS services for CDP Private Cloud Base 7.x + set_fact: + kms_service_type: RANGER_KMS_KTS + when: runtime_parcel_version is version('7.0.0', '>=') + +# These are written out long hand to make future maintenance easier +# This could be rewritten into a module as a good option + +- name: Set service names + ansible.builtin.set_fact: + solr_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(solr_query) }}" + hue_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(hue_query) }}" + knox_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(knox_query) }}" + ranger_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(ranger_query) }}" + oozie_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(oozie_query) }}" + hive_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(hive_query) }}" + hive_on_tez_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(hive_on_tez_query) }}" + spark_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(spark_query) }}" + impala_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(impala_query) }}" + wxm_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(wxm_query) }}" + tp_base_name: "{{ cloudera_manager_mgmt_roles | community.general.json_query(tp_base_query) }}" + tp_mgmt_name: "{{ cloudera_manager_mgmt_roles | community.general.json_query(tp_mgmt_query) }}" + kms_kts_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(kms_kts_query) }}" + kms_service_name: "{{ cloudera_manager_all_services.json | community.general.json_query(kms_query) }}" + vars: + solr_query: "items[?type == 'SOLR'].name | [0]" + hue_query: "items[?type == 'HUE'].name | [0]" + knox_query: "items[?type == 'KNOX'].name | [0]" + ranger_query: "items[?type == 'RANGER'].name | [0]" + oozie_query: "items[?type == 'OOZIE'].name | [0]" + hive_query: "items[?type == 'HIVE'].name | [0]" + hive_on_tez_query: "items[?type == 'HIVE_ON_TEZ'].name | [0]" + spark_query: "items[?type == 'SPARK_ON_YARN'].name | [0]" + impala_query: "items[?type == 'IMPALA'].name | [0]" + kms_kts_query: "items[?type == 'RANGER_KMS_KTS'].name | [0]" + kms_query: "items[?type == '{{ kms_service_type }}'].name | [0]" + wxm_query: "items[?type == 'WXM'].name | [0]" + tp_base_query: "items[?type == 'TELEMETRYPUBLISHER'].name | [0]" + tp_mgmt_query: "items[?roleType == 'TELEMETRYPUBLISHER'].name | [0]" + +- name: Set Service Existence booleans + ansible.builtin.set_fact: + solr_service_exists: "{{ true if solr_service_name != '' else false }}" + hue_service_exists: "{{ true if hue_service_name != '' else false }}" + knox_service_exists: "{{ true if knox_service_name != '' else false }}" + ranger_service_exists: "{{ true if ranger_service_name != '' else false }}" + oozie_service_exists: "{{ true if oozie_service_name != '' else false }}" + hive_service_exists: "{{ true if hive_service_name != '' else false }}" + hive_on_tez_service_exists: "{{ true if hive_on_tez_service_name != '' else false }}" + spark_service_exists: "{{ true if spark_service_name != '' else false }}" + impala_service_exists: "{{ true if impala_service_name != '' else false }}" + kms_kts_service_exists: "{{ true if kms_kts_service_name != '' else false }}" + kms_service_exists: "{{ true if kms_service_name != '' else false }}" + wxm_service_exists: "{{ true if wxm_service_name != '' else false }}" + tp_exists: "{{ false if tp_base_name == '' else true }}" + +# WXM Start + +- name: Get details on WXM Service + when: wxm_service_exists + block: + - name: Get detailed service WXM from CM + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ wxm_service_name | lower }}/roles" + register: cloudera_manager_wxm_all_roles + no_log: yes # overly verbose + + - set_fact: + wxm_api_server: "{{ cloudera_manager_wxm_all_roles.json | community.general.json_query(query) }}" + vars: + query: "items[?type == 'DBUS_API_SERVICE'].hostRef.hostname | [0]" + + - name: Get detailed config WXM from CM + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ wxm_service_name | lower }}/roleConfigGroups/{{ wxm_service_name | lower }}-THUNDERHEAD_SIGMA_CONSOLE-BASE/config?view=full" + register: cloudera_manager_wxm_all_rcgs + no_log: yes # overly verbose + + - set_fact: + wxm_ssl_enabled: "{{ cloudera_manager_wxm_all_roles.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ssl_enabled'].value | [0]" + + - set_fact: + wxm_ssl_enabled: "{{ cloudera_manager_wxm_all_roles.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ssl_enabled'].default | [0]" + when: wxm_ssl_enabled == '' + + - name: Set WXM Databus API server URL + set_fact: + wxm_dbus_api_server_url: "https://{{ wxm_api_server }}:{{ wxm_api_port }}/" + when: wxm_ssl_enabled + + - name: Set WXM Databus API server URL + set_fact: + wxm_dbus_api_server_url: "http://{{ wxm_api_server }}:{{ wxm_api_port }}/" + when: not wxm_ssl_enabled + +# WXM End + +# Ranger Info +- name: Get Ranger Service info + when: ranger_service_exists + block: + - set_fact: + ranger_server: "{{ cm_deployment_services.json | community.general.json_query(query) }}" + vars: + query: "clusters[?name == '{{ cluster_name | urlencode() }}' ].services[].roles[?type == 'RANGER_ADMIN'][].hostRef.hostname | [0]" + + - name: Get Ranger full config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ ranger_service_name | lower }}/config?view=full" + register: full_ranger_config + no_log: yes # overly verbose + + - name: Get Ranger Admin full config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ ranger_service_name | lower }}/roleConfigGroups/{{ ranger_service_name | lower }}-RANGER_ADMIN-BASE/config?view=full" + register: full_ranger_admin_config + no_log: yes # overly verbose + + - set_fact: + ranger_ssl: "{{ full_ranger_admin_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ssl_enabled'].value | [0]" + + - set_fact: + ranger_ssl: "{{ full_ranger_admin_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ssl_enabled'].default | [0]" + when: ranger_ssl == "" + + - set_fact: + ranger_protocol: "{{ 'https' if ranger_ssl else 'http' }}" + + - set_fact: + ranger_port: "{{ full_ranger_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ranger_service_https_port'].value | [0]" + when: ranger_ssl + + - set_fact: + ranger_port: "{{ full_ranger_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ranger_service_https_port'].default | [0]" + when: ranger_ssl and ranger_port == "" + + - set_fact: + ranger_port: "{{ full_ranger_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ranger_service_http_port'].value | [0]" + when: not ranger_ssl + + - set_fact: + ranger_port: "{{ full_ranger_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'ranger_service_http_port'].default | [0]" + when: not ranger_ssl and ranger_port == "" + + - set_fact: + ranger_url: "{{ ranger_protocol }}://{{ ranger_server }}:{{ ranger_port }}" + +# Solr info +- name: Get Solr Service info + when: solr_service_exists + block: + - name: Get SolR roles + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ solr_service_name | lower }}/roles" + register: solr_roles + no_log: yes # overly verbose + + - set_fact: + solr_all_hosts: "{{ solr_roles.json | community.general.json_query(query) }}" + vars: + query: "items[?type == 'SOLR_SERVER'].hostRef.hostname" + + - name: Get SolR full config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ solr_service_name | lower }}/config?view=full" + register: solr_full_config + no_log: yes # overly verbose + + - name: Get SolR full config for SOLR_SERVER-BASE + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ solr_service_name | lower }}/roleConfigGroups/{{ solr_service_name | lower }}-SOLR_SERVER-BASE/config?view=full" + register: solr_full_config_base + no_log: yes # overly verbose + + # Additional solr configs + # Set SoLR protocol + + - set_fact: + solr_ssl: "{{ solr_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_use_ssl'].value | [0]" + + - set_fact: + solr_ssl: "{{ solr_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_use_ssl'].default | [0]" + when: solr_ssl == "" + + - set_fact: + solr_protocol: "{{ 'https' if solr_ssl else 'http' }}" + + # Set SolR port + + - set_fact: + solr_port: "{{ solr_full_config_base.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_https_port'].value | [0]" + when: solr_ssl + + - set_fact: + solr_port: "{{ solr_full_config_base.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_https_port'].default | [0]" + when: solr_ssl and solr_port == '' + + - set_fact: + solr_port: "{{ solr_full_config_base.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_http_port'].value | [0]" + when: not solr_ssl + + - set_fact: + solr_port: "{{ solr_full_config_base.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'solr_http_port'].default | [0]" + when: not solr_ssl and solr_port == '' + + # Set SolR Servers + + - set_fact: + solr_hosts: "{{ solr_roles.json | community.general.json_query(query) }}" + vars: + query: "items[?type == 'SOLR_SERVER'].hostRef.hostname | [0]" + + - set_fact: + solr_url: "{{ solr_protocol }}://{{ solr_hosts }}:{{ solr_port }}" + +- name: Get Knox Service info + when: knox_service_exists + block: + - name: Get Knox Gateway full config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ knox_service_name | lower }}/roleConfigGroups/{{ knox_service_name | lower }}-KNOX_GATEWAY-BASE/config?view=full" + register: knox_full_config + no_log: yes # overly verbose + + - set_fact: + gateway_descriptor_cdp_proxy: "{{ knox_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'gateway_descriptor_cdp_proxy'].value | [0]" + + - set_fact: + gateway_descriptor_cdp_proxy: "{{ knox_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'gateway_descriptor_cdp_proxy'].default | [0]" + when: gateway_descriptor_cdp_proxy == "" + + - set_fact: + gateway_descriptor_cdp_proxy_api: "{{ knox_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'gateway_descriptor_cdp_proxy_api'].value | [0]" + + - set_fact: + gateway_descriptor_cdp_proxy_api: "{{ knox_full_config.json | community.general.json_query(query) }}" + vars: + query: "items[?name == 'gateway_descriptor_cdp_proxy_api'].default | [0]" + when: gateway_descriptor_cdp_proxy_api == "" diff --git a/roles/cloudera_manager/session_timeout/defaults/main.yml b/roles/cloudera_manager/session_timeout/defaults/main.yml new file mode 100644 index 00000000..6bee756b --- /dev/null +++ b/roles/cloudera_manager/session_timeout/defaults/main.yml @@ -0,0 +1,16 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +cloudera_manager_session_timeout: 2592000 diff --git a/roles/cloudera_manager/session_timeout/meta/main.yml b/roles/cloudera_manager/session_timeout/meta/main.yml new file mode 100644 index 00000000..3a5a0054 --- /dev/null +++ b/roles/cloudera_manager/session_timeout/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/session_timeout/tasks/main.yml b/roles/cloudera_manager/session_timeout/tasks/main.yml new file mode 100644 index 00000000..0965cba6 --- /dev/null +++ b/roles/cloudera_manager/session_timeout/tasks/main.yml @@ -0,0 +1,9 @@ +--- + +- name: Set session timeout to 30 days + cloudera.cluster.cm_api: + endpoint: /cm/config + method: PUT + body: "{{ lookup('template', 'unlog.json') }}" + notify: + - restart cloudera-scm-server diff --git a/roles/cloudera_manager/session_timeout/templates/unlog.json b/roles/cloudera_manager/session_timeout/templates/unlog.json new file mode 100644 index 00000000..40c50b9a --- /dev/null +++ b/roles/cloudera_manager/session_timeout/templates/unlog.json @@ -0,0 +1,8 @@ +{ + "items": [ + { + "name": "session_timeout", + "value": "{{ cloudera_manager_session_timeout }}" + } + ] + } \ No newline at end of file diff --git a/roles/cloudera_manager/wait_for_heartbeat/meta/main.yml b/roles/cloudera_manager/wait_for_heartbeat/meta/main.yml index 1aee5eba..524838cc 100644 --- a/roles/cloudera_manager/wait_for_heartbeat/meta/main.yml +++ b/roles/cloudera_manager/wait_for_heartbeat/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index fa4f1c55..6d12f64c 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -19,6 +19,8 @@ tls: False default_cluster_type: base +pvc_type: "" + kms_services: [KEYTRUSTEE, RANGER_KMS, RANGER_KMS_KTS] sdx_services: [ATLAS, HDFS, HIVE, RANGER, SENTRY] diff --git a/roles/config/services/hue_ticket_lifetime/meta/main.yml b/roles/config/services/hue_ticket_lifetime/meta/main.yml new file mode 100644 index 00000000..ea97a5b0 --- /dev/null +++ b/roles/config/services/hue_ticket_lifetime/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file diff --git a/roles/config/services/hue_ticket_lifetime/tasks/main.yml b/roles/config/services/hue_ticket_lifetime/tasks/main.yml new file mode 100644 index 00000000..312b1378 --- /dev/null +++ b/roles/config/services/hue_ticket_lifetime/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Fix Hue ticket lifetime for Free IPA + shell: | + kadmin -p "{{ ipa_admin_user }}" -w "{{ ipaadmin_password }}" -q "modprinc -maxrenewlife 90day +allow_renewable hue/{{ __hue_ticket_item }}@{{ krb5_realm }}" ; + kadmin -p "{{ ipa_admin_user }}" -w "{{ ipaadmin_password }}" -q "modprinc -maxrenewlife 90day krbtgt/{{ krb5_realm }}@{{ krb5_realm }}" ; + loop: "{{ groups['all'] }}" + loop_control: + loop_var: __hue_ticket_item + ignore_errors: true \ No newline at end of file diff --git a/roles/config/services/kms_tls/files/kms_tls.json b/roles/config/services/kms_tls/files/kms_tls.json new file mode 100644 index 00000000..64304f6a --- /dev/null +++ b/roles/config/services/kms_tls/files/kms_tls.json @@ -0,0 +1,29 @@ +{ + "items": [ + { + "name": "ssl_enabled", + "value": "true" + }, + { + "name": "ssl_server_keystore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_client_truststore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_client_truststore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "hadoop_kms_authentication_signer_secret_provider_zookeeper_auth_type", + "value": "kerberos" + } + + ] + } \ No newline at end of file diff --git a/roles/config/services/kms_tls/files/kms_tls_cdh.json b/roles/config/services/kms_tls/files/kms_tls_cdh.json new file mode 100644 index 00000000..fd2e1ea6 --- /dev/null +++ b/roles/config/services/kms_tls/files/kms_tls_cdh.json @@ -0,0 +1,24 @@ +{ + "items": [ + { + "name": "ssl_enabled", + "value": "true" + }, + { + "name": "ssl_server_keystore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_server_keystore_password", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_client_truststore_location", + "value": "{{CM_AUTO_TLS}}" + }, + { + "name": "ssl_client_truststore_password", + "value": "{{CM_AUTO_TLS}}" + } + ] + } \ No newline at end of file diff --git a/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json b/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json new file mode 100644 index 00000000..b514cabe --- /dev/null +++ b/roles/config/services/kms_tls/files/kms_tls_cdh_kms.json @@ -0,0 +1,10 @@ +{ + "items": [ + + { + "name": "hadoop_kms_authentication_signer_secret_provider_zookeeper_auth_type", + "value": "sasl" + } + + ] + } \ No newline at end of file diff --git a/roles/config/services/kms_tls/tasks/main.yml b/roles/config/services/kms_tls/tasks/main.yml new file mode 100644 index 00000000..689b818c --- /dev/null +++ b/roles/config/services/kms_tls/tasks/main.yml @@ -0,0 +1,39 @@ +--- +- name: Push TLS settings for Ranger KMS + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ kms_service_name | lower }}/roleConfigGroups/{{ kms_service_name | lower }}-RANGER_KMS_SERVER_KTS-BASE/config" + method: PUT + body: "{{ lookup('file', 'kms_tls.json', errors='ignore' ) }}" + ignore_errors: yes + when: cloudera_manager_version is version('7.0.0','>=') + +- name: Push TLS settings for Keytrustee roleConfigGroups + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ kms_service_name | lower }}/roleConfigGroups/{{ kms_service_name | lower }}-KMS_KEYTRUSTEE-BASE/config" + method: PUT + body: "{{ lookup('file', 'kms_tls_cdh.json', errors='ignore' ) }}" + ignore_errors: yes + when: cloudera_manager_version is version('7.0.0','<') + +- name: Push TLS settings for Keytrustee config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ kms_service_name | lower }}/config" + method: PUT + body: "{{ lookup('file', 'kms_tls_cdh_kms.json', errors='ignore' ) }}" + ignore_errors: yes + when: cloudera_manager_version is version('7.0.0','<') + +# Restart all clusters to be sure +- name: Find existing clusters + cloudera.cluster.cm_api: + endpoint: /clusters?clusterType=any + register: clusters_response + +- name: Restart clusters and redeploy configs + include_role: + name: cloudera.cluster.operations.restart_cluster + vars: + cluster_to_restart: "{{ __cluster_restart_item }}" + loop: "{{ clusters_response.json | json_query('items[*].name') }}" + loop_control: + loop_var: __cluster_restart_item diff --git a/roles/config/services/oozie_ui/tasks/main.yml b/roles/config/services/oozie_ui/tasks/main.yml new file mode 100644 index 00000000..e8882438 --- /dev/null +++ b/roles/config/services/oozie_ui/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# From: https://docs.cloudera.com/cdp-private-cloud-base/7.1.7/configuring-oozie/topics/oozie-enabling-the-oozie-web-console-on-managed-clusters.html + +- name: Download ext-2.2 file + get_url: + url: http://tiny.cloudera.com/oozie-ext-2.2 + dest: /tmp/ext-2.2.zip + +- name: Remove /var/lib/oozie/ + file: + path: /var/lib/oozie/ + state: absent + ignore_errors: true + +- name: Mkdir /var/lib/oozie/ if it does not exists + file: + path: /var/lib/oozie/ + owner: oozie + group: oozie + state: directory + mode: '0755' + ignore_errors: true + +- name: Install unzip + package: + name: unzip + state: latest + +- name: Unzip ext-2.2 file + shell: rm -rf /var/lib/oozie/* + ignore_errors: true + +- name: Unzip ext-2.2 file + shell: unzip /tmp/ext-2.2.zip -d /var/lib/oozie/ + ignore_errors: true + +- name: Chown ext-2.2 + shell: chown -R oozie:oozie /var/lib/oozie/ext-2.2 + ignore_errors: true + diff --git a/roles/config/services/ranger_pvc_default_policies/meta/main.yml b/roles/config/services/ranger_pvc_default_policies/meta/main.yml new file mode 100644 index 00000000..0d623502 --- /dev/null +++ b/roles/config/services/ranger_pvc_default_policies/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.services_info diff --git a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json new file mode 100644 index 00000000..d66cecd6 --- /dev/null +++ b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_hadoop.json @@ -0,0 +1,90 @@ +{ + "id": "1", + "guid": "...", + "isEnabled": true, + "version": 1, + "service": "cm_hdfs", + "name": "Audit and tmp policies for internal users", + "policyType": 0, + "policyPriority": 0, + "description": "Policy for all users from hadoop group", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/user", + "/tmp", + "/ranger/audit" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + }, + { + "type": "execute", + "isAllowed": true + } + ], + "users": [ + "solr", + "kafka", + "hive", + "hdfs", + "impala", + "knox", + "yarn", + "atlas", + "ozone", + "francois" + ], + "groups": [ + "hadoop" + ], + "roles": [ + + ], + "conditions": [ + + ], + "delegateAdmin": true + } + ], + "denyPolicyItems": [ + + ], + "allowExceptions": [ + + ], + "denyExceptions": [ + + ], + "dataMaskPolicyItems": [ + + ], + "rowFilterPolicyItems": [ + + ], + "serviceType": "hdfs", + "options": { + + }, + "validitySchedules": [ + + ], + "policyLabels": [ + + ], + "zoneName": "", + "isDenyAllElse": false + } \ No newline at end of file diff --git a/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json new file mode 100644 index 00000000..cc3808d4 --- /dev/null +++ b/roles/config/services/ranger_pvc_default_policies/policies/hdfs_user.json @@ -0,0 +1,78 @@ +{ + "id": "1", + "guid": "...", + "isEnabled": true, + "version": 1, + "service": "cm_hdfs", + "name": "Policy fo Users on their Home Directory", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "path": { + "values": [ + "/user/{USER}" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + }, + { + "type": "execute", + "isAllowed": true + } + ], + "users": [ + "{USER}" + ], + "groups": [ + ], + "roles": [ + + ], + "conditions": [ + + ], + "delegateAdmin": true + } + ], + "denyPolicyItems": [ + + ], + "allowExceptions": [ + + ], + "denyExceptions": [ + + ], + "dataMaskPolicyItems": [ + + ], + "rowFilterPolicyItems": [ + + ], + "serviceType": "hdfs", + "options": { + + }, + "validitySchedules": [ + + ], + "policyLabels": [ + + ], + "zoneName": "", + "isDenyAllElse": false + } \ No newline at end of file diff --git a/roles/config/services/ranger_pvc_default_policies/policies/solr.json b/roles/config/services/ranger_pvc_default_policies/policies/solr.json new file mode 100644 index 00000000..7cf5589a --- /dev/null +++ b/roles/config/services/ranger_pvc_default_policies/policies/solr.json @@ -0,0 +1,85 @@ +{ + "id": 1, + "guid": "...", + "isEnabled": true, + "version": 1, + "service": "cm_solr", + "name": "SolR all collections", + "policyType": 0, + "policyPriority": 0, + "description": "Policy for admin users on collections", + "isAuditEnabled": true, + "resources": { + "collection": { + "values": [ + "*", + "tmp_*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "query", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "others", + "isAllowed": true + }, + { + "type": "solr_admin", + "isAllowed": true + } + ], + "users": [ + "solr", + "hue" + ], + "groups": [ + + ], + "roles": [ + + ], + "conditions": [ + + ], + "delegateAdmin": true + } + ], + "denyPolicyItems": [ + + ], + "allowExceptions": [ + + ], + "denyExceptions": [ + + ], + "dataMaskPolicyItems": [ + + ], + "rowFilterPolicyItems": [ + + ], + "serviceType": "solr", + "options": { + + }, + "validitySchedules": [ + + ], + "policyLabels": [ + + ], + "zoneName": "", + "isDenyAllElse": false + } \ No newline at end of file diff --git a/roles/config/services/ranger_pvc_default_policies/tasks/main.yml b/roles/config/services/ranger_pvc_default_policies/tasks/main.yml new file mode 100644 index 00000000..e98e3b5f --- /dev/null +++ b/roles/config/services/ranger_pvc_default_policies/tasks/main.yml @@ -0,0 +1,20 @@ +--- + +- name: Post Ranger policies declared in policies directory + register: __ranger_pol_response + uri: + url: "{{ ranger_url }}/service/public/v2/api/policy" + method: POST + user: "{{ ranger_user }}" + password: "{{ ranger_password }}" + return_content: yes + body: "{{ lookup('template', '{{ item.src }}' ) }}" + body_format: json + status_code: 200 + validate_certs: no + force_basic_auth: yes + no_log: True + with_filetree: "{{ role_path }}/policies" + failed_when: + - __ranger_pol_response is failed + - "'Another policy already exists for this name' not in __ranger_pol_response.json.msgDesc" diff --git a/roles/config/services/solr_knox/meta/main.yml b/roles/config/services/solr_knox/meta/main.yml new file mode 100644 index 00000000..0d623502 --- /dev/null +++ b/roles/config/services/solr_knox/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.services_info diff --git a/roles/config/services/solr_knox/tasks/add_solr_knox_host.yml b/roles/config/services/solr_knox/tasks/add_solr_knox_host.yml new file mode 100644 index 00000000..ba4e32e5 --- /dev/null +++ b/roles/config/services/solr_knox/tasks/add_solr_knox_host.yml @@ -0,0 +1,19 @@ +--- +- set_fact: + new_gateway_descriptor_cdp_proxy: "{{ gateway_descriptor_cdp_proxy }}#SOLR:url={{ solr_protocol }}://{{ solr_host }}:{{ solr_port }}/solr/" + +- set_fact: + new_gateway_descriptor_cdp_proxy_api: "{{ gateway_descriptor_cdp_proxy_api }}#SOLR:url={{ solr_protocol }}://{{ solr_host }}:{{ solr_port }}/solr/" + +- name: Push new Knox config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ knox_service_name | lower }}/roleConfigGroups/{{ knox_service_name | lower }}-KNOX_GATEWAY-BASE/config" + method: PUT + body: "{{ lookup('template', 'solr_knox_url.json' ) }}" + +- name: Push new Knox API config + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ knox_service_name | lower }}/roleConfigGroups/{{ knox_service_name | lower }}-KNOX_GATEWAY-BASE/config" + method: PUT + body: "{{ lookup('template', 'solr_knox_url_api.json' ) }}" + register: __add_solr_knox diff --git a/roles/config/services/solr_knox/tasks/main.yml b/roles/config/services/solr_knox/tasks/main.yml new file mode 100644 index 00000000..44a81086 --- /dev/null +++ b/roles/config/services/solr_knox/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Add Solr host to Knox +- name: add solr host in config + include_tasks: add_solr_knox_host.yml + loop: "{{ solr_all_hosts }}" + loop_control: + loop_var: solr_host + when: solr_all_hosts | length > 0 + +# Restart Knox +- name: Restart Knox + cloudera.cluster.cm_api: + method: POST + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ knox_service_name | lower }}/commands/restart" + when: + - __add_solr_knox is defined + - __add_solr_knox.changed diff --git a/roles/config/services/solr_knox/templates/solr_knox_url.json b/roles/config/services/solr_knox/templates/solr_knox_url.json new file mode 100644 index 00000000..1267b807 --- /dev/null +++ b/roles/config/services/solr_knox/templates/solr_knox_url.json @@ -0,0 +1,8 @@ +{ + "items": [ + { + "name": "gateway_descriptor_cdp_proxy", + "value": "{{ new_gateway_descriptor_cdp_proxy }}" + } + ] + } \ No newline at end of file diff --git a/roles/config/services/solr_knox/templates/solr_knox_url_api.json b/roles/config/services/solr_knox/templates/solr_knox_url_api.json new file mode 100644 index 00000000..6f9f7926 --- /dev/null +++ b/roles/config/services/solr_knox/templates/solr_knox_url_api.json @@ -0,0 +1,8 @@ +{ + "items": [ + { + "name": "gateway_descriptor_cdp_proxy_api", + "value": "{{ new_gateway_descriptor_cdp_proxy_api }}" + } + ] + } \ No newline at end of file diff --git a/roles/config/services/solr_ranger_plugin/meta/main.yml b/roles/config/services/solr_ranger_plugin/meta/main.yml new file mode 100644 index 00000000..0d623502 --- /dev/null +++ b/roles/config/services/solr_ranger_plugin/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.services_info diff --git a/roles/config/services/solr_ranger_plugin/tasks/main.yml b/roles/config/services/solr_ranger_plugin/tasks/main.yml new file mode 100644 index 00000000..aff71252 --- /dev/null +++ b/roles/config/services/solr_ranger_plugin/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Create in Ranger the SolR service + register: __ranger_solr_plugin + uri: + url: "{{ ranger_url }}/service/public/v2/api/service" + method: POST + user: "{{ ranger_user }}" + password: "{{ ranger_password }}" + return_content: yes + body: "{{ lookup('template', 'solr_plugin.json' ) }}" + body_format: json + status_code: 200 + validate_certs: no + force_basic_auth: yes + no_log: yes + failed_when: + - __ranger_solr_plugin is failed + - "'Duplicate service name' not in __ranger_solr_plugin.json.msgDesc" + +# Restart SolR + +- name: Restart SolR + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services/{{ solr_service_name | lower }}/commands/restart" + when: __ranger_solr_plugin.changed diff --git a/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json b/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json new file mode 100644 index 00000000..c3e13c15 --- /dev/null +++ b/roles/config/services/solr_ranger_plugin/templates/solr_plugin.json @@ -0,0 +1,19 @@ +{ + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin", + "type": "solr", + "name": "cm_solr", + "displayName": "cm_solr", + "description": "", + "tagService": "cm_tag", + "configs": { + "username": "admin", + "password": "{{ solr_admin_password }}", + "solr.url": "{{ solr_url }}/solr", + "commonNameForCertificate": "", + "tag.download.auth.users": "solr", + "policy.download.auth.users": "solr", + "ranger.plugin.audit.filters": "[{'accessResult':'DENIED','isAudited':true},{'users':['hive','hdfs','kafka','hbase','solr','rangerraz','knox','atlas'],'isAudited':false}]" + } + } \ No newline at end of file diff --git a/roles/deployment/cluster/tasks/create_base.yml b/roles/deployment/cluster/tasks/create_base.yml index ac080232..83fae426 100644 --- a/roles/deployment/cluster/tasks/create_base.yml +++ b/roles/deployment/cluster/tasks/create_base.yml @@ -31,7 +31,7 @@ mode: 0600 #when: cluster_template_dry_run -- name: Import cluster template +- name: Import cluster template for {{ cluster.name }} cloudera.cluster.cm_api: endpoint: /cm/importClusterTemplate?addRepositories=true method: POST diff --git a/roles/deployment/cluster/tasks/create_ecs.yml b/roles/deployment/cluster/tasks/create_ecs.yml index 218193bb..e7071ef6 100644 --- a/roles/deployment/cluster/tasks/create_ecs.yml +++ b/roles/deployment/cluster/tasks/create_ecs.yml @@ -25,7 +25,6 @@ services: "{{ cluster.ecs_databases | default(ecs_databases) }}" # Add PVC parcel repo - assume that this has been set via REMOTE_PARCEL_REPO_URLS in the cm config. We can add this in later if needed - - name: Create ECS cluster cloudera.cluster.cm_api: endpoint: /clusters @@ -119,14 +118,20 @@ delay: "{{ parcel_poll_duration | default(60) }}" - name: Generate custom values + when: cluster.controlplane_config.Database.Mode != 'embedded' set_fact: custom_values: "{{ lookup('template', 'cluster_template/ecs/controlPlaneValues.j2') | from_yaml | combine(cluster.controlplane_config, recursive=True) }}" +- name: Generate custom values - Embedded + when: cluster.controlplane_config.Database.Mode == 'embedded' + set_fact: + custom_values: "{{ lookup('template', 'cluster_template/ecs/controlPlaneValuesEmbedded.j2') | from_yaml | combine(cluster.controlplane_config, recursive=True) }}" + - name: Show custom values debug: var: custom_values -- name: Install ECS cluster +- name: Install ECS Embedded Control Plane cloudera.cluster.cm_api: endpoint: /controlPlanes/commands/installEmbeddedControlPlane method: POST diff --git a/roles/deployment/cluster/tasks/create_kts.yml b/roles/deployment/cluster/tasks/create_kts.yml index e2e0303e..f756bdbe 100644 --- a/roles/deployment/cluster/tasks/create_kts.yml +++ b/roles/deployment/cluster/tasks/create_kts.yml @@ -26,7 +26,7 @@ - block: - - name: Import cluster template + - name: Import cluster template for {{ cluster.name }} cloudera.cluster.cm_api: endpoint: /cm/importClusterTemplate?addRepositories=true method: POST diff --git a/roles/deployment/cluster/tasks/main.yml b/roles/deployment/cluster/tasks/main.yml index 11b4a39d..9afdc39f 100644 --- a/roles/deployment/cluster/tasks/main.yml +++ b/roles/deployment/cluster/tasks/main.yml @@ -105,6 +105,7 @@ - cluster.type | default(default_cluster_type) == 'compute' - cluster.name not in existing_clusters +# This process is not idempotent, as the whole ECS setup process must succeed or it will skip on subsequent runs - name: Create ECS clusters include_tasks: create_ecs.yml loop: "{{ definition.clusters }}" @@ -116,7 +117,6 @@ - cluster.type | default(default_cluster_type) == 'ecs' - cluster.name not in existing_clusters - - name: Restart Cloudera Management Service cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart diff --git a/roles/deployment/cluster/templates/cluster_template/deployment.j2 b/roles/deployment/cluster/templates/cluster_template/deployment.j2 deleted file mode 100644 index 5a2f0e87..00000000 --- a/roles/deployment/cluster/templates/cluster_template/deployment.j2 +++ /dev/null @@ -1,14 +0,0 @@ -{% import 'cm_api.j2' as cm_api with context %} -{ -{% set cluster_type = cluster.type | default('base') %} -{% set template_dir = (cluster_type == 'kts') | ternary('kts', 'base') %} - "cdhVersion" : "{{ cloudera_runtime_version }}", - "displayName" : "{{ cluster.name }}", - "cmVersion" : "{{ cloudera_manager_version }}", - "repositories" : {{ cluster.repositories | to_json }}, - "products" : {{ products | to_json }}, - "services" : {% include 'cluster_template/' + template_dir + '/services.j2' %}, - "hostTemplates" : {%- include 'cluster_template/' + template_dir + '/hostTemplates.j2' -%}, - "instantiator" : {%- include 'cluster_template/' + template_dir + '/instantiator.j2' -%}{%- if cloudera_manager_version is version('6.2.0','>=') -%}, - "clusterSpec" : {%- include 'cluster_template/common/clusterSpec.j2' -%}{%- endif -%} -} diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 index b438561e..bb6cfc84 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 @@ -4,7 +4,7 @@ { "name" : "{{ cluster.name }}", "displayName": "{{ cluster.name }}", - "version" : "EXPERIENCE1", + "version" : "DATA_SERVICES1", "fullVersion": "{{ products | cloudera.cluster.get_product_version('ECS') }}", "clusterType": "EXPERIENCE_CLUSTER" } diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValuesEmbedded.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValuesEmbedded.j2 new file mode 100644 index 00000000..c3c7a0ce --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValuesEmbedded.j2 @@ -0,0 +1,82 @@ +Services: + thunderheadenvironment: + Config: + database: + name: "db-env" + clusterproxy: + Config: + database: + name: "db-clusterproxy" + classicclusters: + Config: + database: + name: "db-classicclusters" + mlxcontrolplaneapp: + Config: + database: + name: "db-mlx" + cpxliftie: + Config: + database: + name: "db-liftie" + monitoringapp: + Config: + database: + name: "db-alerts" + dwx: + Config: + database: + name: "db-dwx" + dex: + Config: + database: + name: "db-dex" + resourcepoolmanager: + Config: + database: + name: "db-resourcepoolmanager" + clusteraccessmanager: + Config: + database: + name: "db-clusteraccessmanager" + thunderheadusermanagementprivate: + Config: + database: + name: "db-ums" + replicaCount: 2 + + thunderheadiamconsole: + Config: + replicaCount: 2 + + thunderheadiamapi: + Config: + replicaCount: 2 + +ContainerInfo: + Mode: embedded + CopyDocker: true + +Database: + Mode: embedded + EmbeddedDbStorage: 50 + +Vault: + Mode: embedded + EmbeddedStorage: 2 + +OptionalCerts: + MiscCaCerts: "" + +Other: + cmLicense: + uuid: {{ cloudera_manager_repo_username }} + owner: "{{ cloudera_manager_license | regex_replace('(.|\n)*\"name\"\\s*:\\s*\"([^\"]*)\"(.|\n)*', '\\2') }}" + +Platform: + Name: standard + +ApplicationDomain: "{{ cluster.application_domain }}" + +SecurityContext: + Enabled: true diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 index 10fc1de5..322f96f4 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 @@ -1,7 +1,7 @@ { "items" : [ {%- set host_joiner = joiner(",") -%} -{%- for host in groups['ecs_servers'] -%} +{%- for host in groups['ecs_nodes'] -%} {{ host_joiner() }} { "hostId" : "{{ cloudera_manager_api_hosts[host]['id'] }}" diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 index 39859383..e5e6aa97 100644 --- a/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 +++ b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 @@ -19,7 +19,7 @@ {%- for (template_service, roles) in role_mapping.items() -%} {%- if template_service == service -%} {%- for role in roles -%} - {%- for host in groups[template_name] -%} + {%- for host in groups["host_template_" + template_name] -%} {{ role_sep() }} { "type": "{{ role }}", diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index 21547c9d..729547b4 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -13,17 +13,27 @@ # limitations under the License. --- -database_host: "{{ groups['db_server'][0] | default('localhost') }}" -database_type: postgresql +database_host: "{{ groups['db_server'][0] | default('cloudera_manager[0]') }}" + + database_default_password: changeme -database_version: "{{ 10 if database_type == 'postgresql' else 10.2 }}" database_tls: false +database_type: postgresql +database_version: "{{ default_database_versions[database_type][ansible_distribution_major_version] }}" +default_database_versions: + postgresql: + '7': 10 + '8': 12 + mariadb: + '7': 10.2 + '8': 10.2 -krb5_realm: CLOUDERA.LOCAL -krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" -krb5_kdc_admin_password: changeme -krb5_kdc_type: MIT KDC -krb5_enc_types: "aes256-cts aes128-cts" +# Located in cloudera.cluster.infrastructure.krb5_common +#krb5_realm: CLOUDERA.LOCAL +#krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" +#krb5_kdc_admin_password: "{{ admin_password }}" +#krb5_kdc_type: MIT KDC +#krb5_enc_types: "aes256-cts aes128-cts" manual_tls_cert_distribution: false local_temp_dir: '/tmp' diff --git a/roles/deployment/definition/meta/main.yml b/roles/deployment/definition/meta/main.yml new file mode 100644 index 00000000..3892ffe2 --- /dev/null +++ b/roles/deployment/definition/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - cloudera.cluster.infrastructure.krb5_common \ No newline at end of file diff --git a/roles/deployment/definition/tasks/main.yml b/roles/deployment/definition/tasks/main.yml index eb49f216..d5e815af 100644 --- a/roles/deployment/definition/tasks/main.yml +++ b/roles/deployment/definition/tasks/main.yml @@ -18,7 +18,7 @@ ansible.builtin.set_fact: _host_template_cluster_map: "{{ lookup('template', './template_cluster_map.j2') | from_yaml }}" vars: - clusters: "{{ clusters | default([]) }}" + clusters: "{{ definition.clusters | default([]) }}" - name: Generate definition canonical structure from supplied Definition details ansible.builtin.set_fact: diff --git a/roles/deployment/groupby/tasks/main.yml b/roles/deployment/groupby/tasks/main.yml index 3aa15dc8..686be889 100644 --- a/roles/deployment/groupby/tasks/main.yml +++ b/roles/deployment/groupby/tasks/main.yml @@ -46,12 +46,12 @@ loop_var: role when: host_template is defined +- name: Group hosts based on whether TLS flag is set in inventory + group_by: + key: "{{ 'tls' if tls | default(False) else 'no_tls' }}" + - name: (debug) Show derived groups delegate_to: localhost debug: var: groups verbosity: 3 - -- name: Group hosts based on whether TLS flag is set in inventory - group_by: - key: "{{ 'tls' if tls | default(False) else 'no_tls' }}" diff --git a/roles/deployment/services/kts_common/defaults/main.yml b/roles/deployment/services/kts_common/defaults/main.yml index 22ef282b..7f74489e 100644 --- a/roles/deployment/services/kts_common/defaults/main.yml +++ b/roles/deployment/services/kts_common/defaults/main.yml @@ -22,8 +22,15 @@ keytrustee_server_key_files: - gpg.conf - keytrustee.conf - logging.conf + - trustdb.gpg + +# GnuPG 2.1+ uses .kbx for keyring, and retired secring / random_seed +keytrustee_server_gpg_files: + - secring.gpg - pubring.gpg - pubring.gpg~ - random_seed - - secring.gpg - - trustdb.gpg + +keytrustee_server_kbx_files: + - pubring.kbx + - pubring.kbx~ \ No newline at end of file diff --git a/roles/deployment/services/kts_high_availability/tasks/main.yml b/roles/deployment/services/kts_high_availability/tasks/main.yml index 1ea6b663..e909da61 100644 --- a/roles/deployment/services/kts_high_availability/tasks/main.yml +++ b/roles/deployment/services/kts_high_availability/tasks/main.yml @@ -33,12 +33,18 @@ state: directory mode: 0777 +# GnuPG 2.1+ uses .kbx for keyring, and retired secring / random_seed +- name: Determine gnupg version + delegate_to: "{{ groups.kts_active | first }}" + register: __gnupg_version + shell: "gpg2 --version | head -n 1 | rev | cut -d ' ' -f1 | rev" + - name: Fetch GPG keys and configs from active Key Trustee Server delegate_to: "{{ groups.kts_active | first }}" fetch: src: "{{ keytrustee_server_conf_dir }}/{{ item }}" dest: "{{ local_temp_dir }}/kts" - loop: "{{ keytrustee_server_key_files }}" + loop: "{{ keytrustee_server_key_files + (keytrustee_server_kbx_files if __gnupg_version.stdout is version('2.1', '>=') else keytrustee_server_gpg_files) }}" - name: Copy to passive Key Trustee Server delegate_to: "{{ groups.kts_passive | first }}" diff --git a/roles/deployment/services/wxm/defaults/main.yml b/roles/deployment/services/wxm/defaults/main.yml new file mode 100644 index 00000000..960eefa6 --- /dev/null +++ b/roles/deployment/services/wxm/defaults/main.yml @@ -0,0 +1,2 @@ +altus_key_id: '' +altus_private_key: '' \ No newline at end of file diff --git a/roles/deployment/services/wxm/meta/main.yml b/roles/deployment/services/wxm/meta/main.yml new file mode 100644 index 00000000..0d623502 --- /dev/null +++ b/roles/deployment/services/wxm/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.services_info diff --git a/roles/deployment/services/wxm/tasks/configure_telemetry.yml b/roles/deployment/services/wxm/tasks/configure_telemetry.yml new file mode 100644 index 00000000..5fee561f --- /dev/null +++ b/roles/deployment/services/wxm/tasks/configure_telemetry.yml @@ -0,0 +1,56 @@ +--- + +# Add access key for Altus to base CM + +- name: Set Altus private Key into one line + set_fact: + altus_private_key_one_line: "{{ lookup('file', altus_private_key ) | replace('\n', '\\n') | replace('\"', '\\\"' ) }}" + +- name: DEBUG - altus_private_key_one_line + debug: + msg: "{{ altus_private_key_one_line }}" + when: debug | default(false) + +- name: Delete Altus account if it already exists + cloudera.cluster.cm_api: + endpoint: "/externalAccounts/delete/altus-key-for-wxm" + method: DELETE + ignore_errors: true + +- name: Create Altus Account + cloudera.cluster.cm_api: + endpoint: "/externalAccounts/create" + method: POST + body: "{{ lookup('template', 'add_altus_key.json') }}" + + +# Get host Id of this host as it is required to add it as the TP host +- set_fact: + tp_host_id: "{{ hosts_details.json | community.general.json_query(query) }}" + vars: + query: "items[?hostname == '{{ inventory_hostname }}'].hostId | [0]" + when: not tp_exists + +- name: Add Telemetry Publisher to CM + cloudera.cluster.cm_api: + endpoint: "/cm/service/roles" + method: POST + body: "{{ lookup('template', 'add_telemetry.json') }}" + timeout: 120 + when: not tp_exists + +- name: Refresh CM Services Info + include_role: cloudera.cluster.cloudera_manager.services_info + +- name: Set WXM reference for telemetry publisher + cloudera.cluster.cm_api: + endpoint: "/cm/service/roleConfigGroups/{{ tp_mgmt_name }}/config" + method: PUT + body: "{{ lookup('template', 'wxm_config.json') }}" + +# Reference access key to telemetry publisher +- name: Set Altus account reference to Telemetry + cloudera.cluster.cm_api: + endpoint: "/cm/config" + method: PUT + body: "{{ lookup('template', 'altus_key_config.json') }}" diff --git a/roles/deployment/services/wxm/tasks/main.yml b/roles/deployment/services/wxm/tasks/main.yml new file mode 100644 index 00000000..3102b82b --- /dev/null +++ b/roles/deployment/services/wxm/tasks/main.yml @@ -0,0 +1,33 @@ +--- + +- assert: + that: + - altus_private_key | length > 0 + - altus_key_id | length > 0 + quiet: True + fail_msg: >- + Altus key id and private key must be provided to configure Telemetry service for WXM + +- name: Configure Telemetry + ansible.builtin.include_tasks: configure_telemetry.yml + +- name: Configure Truststore in Agent for base + ansible.builtin.include_tasks: truststore_to_base.yml + +- name: Restart Telemetry Publisher + include_role: + name: cloudera.cluster.operations.restart_mgmt_services + vars: + services_to_restart: + - mgmt-TELEMETRYPUBLISHER + +- name: Restart Base Cluster Services + include_role: + name: cloudera.cluster.operations.restart_cluster_services + vars: + services_to_restart: + - HIVE + - HIVE_ON_TEZ + - SPARK_ON_YARN + - IMPALA + - WXM \ No newline at end of file diff --git a/roles/deployment/services/wxm/tasks/truststore_to_base.yml b/roles/deployment/services/wxm/tasks/truststore_to_base.yml new file mode 100644 index 00000000..cdd1bcf0 --- /dev/null +++ b/roles/deployment/services/wxm/tasks/truststore_to_base.yml @@ -0,0 +1,21 @@ +--- +- name: Get truststore content into a file + cloudera.cluster.cm_api: + endpoint: "/certs/truststore?type=pem" + body_format: json + dest: "/tmp/wxm_truststore.pem" + +- name: Get Base truststore password + cloudera.cluster.cm_api: + endpoint: "/certs/truststorePassword" + register: cloudera_manager_truststore_password_api + no_log: True + +# Import CA from WXM cluster and import it into base truststore +- name: Import truststore into Base truststore + shell: "echo 'yes' | keytool -import -alias wxm_truststore -keystore /var/lib/cloudera-scm-agent/agent-cert/cm-auto-global_truststore.jks -file /tmp/wxm_truststore.pem -storepass {{ cloudera_manager_truststore_password_api.content }}" + ignore_errors: true + +- name: Import truststore into Base truststore + shell: "echo 'yes' | keytool -import -alias wxm_truststore -keystore /var/lib/cloudera-scm-agent/agent-cert/cm-auto-in_cluster_truststore.jks -file /tmp/wxm_truststore.pem -storepass {{ cloudera_manager_truststore_password_api.content }}" + ignore_errors: true diff --git a/roles/deployment/services/wxm/templates/add_altus_key.json b/roles/deployment/services/wxm/templates/add_altus_key.json new file mode 100644 index 00000000..1eb4574c --- /dev/null +++ b/roles/deployment/services/wxm/templates/add_altus_key.json @@ -0,0 +1,19 @@ +{ + "typeName":"ALTUS_ACCESS_KEY_AUTH", + "displayName":"altus-key-for-wxm", + "name":"altus-key-for-wxm", + "accountConfigs": + { + "items": + [ + { + "name":"access_key_id", + "value":"{{ altus_key_id }}" + }, + { + "name":"private_key", + "value":"{{ altus_private_key_one_line }}" + } + ] + } +} diff --git a/roles/deployment/services/wxm/templates/add_telemetry.json b/roles/deployment/services/wxm/templates/add_telemetry.json new file mode 100644 index 00000000..bd267eb7 --- /dev/null +++ b/roles/deployment/services/wxm/templates/add_telemetry.json @@ -0,0 +1,12 @@ +{ + "items" : [ + { + "name" : "mgmt-TELEMETRYPUBLISHER", + "type" : "TELEMETRYPUBLISHER", + "hostRef" : { + "hostId" : "{{ tp_host_id }}", + "hostname" : "{{ tp_host }}" + } + } + ] +} \ No newline at end of file diff --git a/roles/deployment/services/wxm/templates/altus_key_config.json b/roles/deployment/services/wxm/templates/altus_key_config.json new file mode 100644 index 00000000..ba513f0f --- /dev/null +++ b/roles/deployment/services/wxm/templates/altus_key_config.json @@ -0,0 +1,8 @@ +{ + "items": [ + { + "name": "telemetry_altus_account", + "value": "altus-key-for-wxm" + } + ] +} \ No newline at end of file diff --git a/roles/deployment/services/wxm/templates/wxm_config.json b/roles/deployment/services/wxm/templates/wxm_config.json new file mode 100644 index 00000000..4a18bac8 --- /dev/null +++ b/roles/deployment/services/wxm/templates/wxm_config.json @@ -0,0 +1,8 @@ +{ + "items": [ + { + "name": "telemetrypublisher_safety_valve", + "value": "telemetry.upload.job.logs=true\ntelemetry.altus.url={{ wxm_dbus_api_server_url }}" + } + ] + } \ No newline at end of file diff --git a/roles/infrastructure/custom_repo/defaults/main.yml b/roles/infrastructure/custom_repo/defaults/main.yml index 90bcb158..416cf7f3 100644 --- a/roles/infrastructure/custom_repo/defaults/main.yml +++ b/roles/infrastructure/custom_repo/defaults/main.yml @@ -19,5 +19,4 @@ repo_tar_local_dir: repo repo_tar_files: "{{ definition.repo_tar_files | default([]) }}" keep_newer: yes -cm_repo_tarball_url: "{{ definition.cm_repo_tarball_url | default('') }}" -custom_repo_rehost_files: "{{ definition.custom_repo_rehost_files | default([]) }}" +custom_repo_rehost_files: "{{ definition.custom_repo_rehost_files | default([]) }}" \ No newline at end of file diff --git a/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml b/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml index f94dbbf8..4e52182a 100644 --- a/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml +++ b/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml @@ -31,7 +31,7 @@ poll: 0 ansible.builtin.get_url: url: "{{ __parcel_download_item }}" - dest: "/var/www/html/{{ __parcel_download_item | urlsplit('path') }}" + dest: "/var/www/html{{ __parcel_download_item | urlsplit('path') }}" - name: Track async downloads to completion [ This may take a while if your files are very large or far away ] loop: "{{ __infra_download_parcels_results.results }}" @@ -65,14 +65,3 @@ src: "/var/www/html{{ __tmp_unpack_item | urlsplit('path') }}" dest: "/var/www/html{{ __tmp_unpack_item | urlsplit('path') | regex_replace('^(.+)repo.+-(.+)\\.tar\\.gz$', '\\1\\2' + '/yum/') }}" keep_newer: "{{ keep_newer }}" - -- name: Set Cloudera Manager Base Repo if included in rehosting list - when: "{{ custom_repo_rehost_files | select('search', 'tar.gz') | list | select('search', '/cm') | list }} | length > 0" - ansible.builtin.set_fact: - cloudera_archive_base_url: "http://{{ groups['custom_repo'] | first }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.cloudera_manager + groups.cluster }}" - loop_control: - loop_var: __play_host - label: __play_host diff --git a/roles/infrastructure/krb5_client/files/dbus_session.conf b/roles/infrastructure/krb5_client/files/dbus_session.conf new file mode 100644 index 00000000..9032e606 --- /dev/null +++ b/roles/infrastructure/krb5_client/files/dbus_session.conf @@ -0,0 +1,6 @@ + + + 360000 + 360000 + \ No newline at end of file diff --git a/roles/infrastructure/krb5_client/tasks/freeipa.yml b/roles/infrastructure/krb5_client/tasks/freeipa.yml index 7621424b..7097e9a9 100644 --- a/roles/infrastructure/krb5_client/tasks/freeipa.yml +++ b/roles/infrastructure/krb5_client/tasks/freeipa.yml @@ -13,15 +13,39 @@ # limitations under the License. --- +- name: Fix FreeIPA Dbus configuration + include_tasks: freeipa_dbus_patch.yml + when: + - ansible_os_family == 'RedHat' + +- name: Set hosts to point to FreeIPA DNS when requested + when: freeipa_autodns | default(false) + include_tasks: freeipa_autodns.yml + - name: Setup FreeIPA Client ansible.builtin.include_role: name: freeipa.ansible_freeipa.ipaclient vars: state: present ipaserver_realm: "{{ krb5_realm }}" - ipaserver_domain: "{{ krb5_domain }}" + ipaserver_domain: "{{ krb5_domain | default(krb5_realm | lower) }}" ipaclient_servers: "{{ groups['krb5_server'] }}" - when: "krb5_kdc_type == 'Red Hat IPA' and 'krb5_server' in groups" + +- name: Include Private Cloud config changes + ansible.builtin.include_tasks: pvc_configs.yml + when: + - pvc_type is defined + - pvc_type == 'OC' or pvc_type == 'ECS' + +- name: Set sssd to enumerate users and groups + when: use_sssd_for_enumeration is defined and use_sssd_for_enurmeration == True + lineinfile: + path: /etc/sssd/sssd.conf + insertafter: "^\\[domain/.+\\]" + regexp: "^enumerate" + line: "enumerate = True" + notify: + - restart sssd - name: Set up renew_lifetime in krb5.conf lineinfile: @@ -29,7 +53,7 @@ insertafter: "^\\[libdefaults\\]" regexp: "^ renew_lifetime" line: " renew_lifetime = 7d" - when: + when: - krb5_kdc_type == 'Red Hat IPA' - "'cluster' in group_names or 'cloudera_manager' in group_names" @@ -38,6 +62,6 @@ path: /etc/krb5.conf regexp: "^ default_ccache_name" state: absent - when: + when: - krb5_kdc_type == 'Red Hat IPA' - "'cluster' in group_names or 'cloudera_manager' in group_names" diff --git a/roles/infrastructure/krb5_client/tasks/freeipa_autodns.yml b/roles/infrastructure/krb5_client/tasks/freeipa_autodns.yml new file mode 100644 index 00000000..5882f611 --- /dev/null +++ b/roles/infrastructure/krb5_client/tasks/freeipa_autodns.yml @@ -0,0 +1,73 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Configure autodns on FreeIPA for el7 or el8 + when: + - ansible_distribution_major_version | int > 6 + - ansible_os_family == 'RedHat' + block: + - name: Gather facts from KRB5 Server + ansible.builtin.setup: + gather_subset: + - 'default_ipv4' + delegate_to: "{{ krb5_ip_collect_item }}" + delegate_facts: true + loop: "{{ groups['krb5_server'] }}" + loop_control: + loop_var: krb5_ip_collect_item + + - set_fact: + krb5_server_ip: "{{ hostvars[groups.krb5_server | first].ansible_facts.default_ipv4.address }}" + + - name: Insert FreeIPA server as first dns nameserver + ansible.builtin.lineinfile: + path: /etc/resolv.conf + insertbefore: BOF + line: "nameserver {{ krb5_server_ip }}" + + - name: Ensure directory for NetworkManager override exists + file: + path: /etc/NetworkManager/conf.d/ + state: directory + recurse: yes + + - name: Ensure dns configuration persists through reboot + ansible.builtin.copy: + content: | + [main] + dns=none + dest: /etc/NetworkManager/conf.d/disable-resolve.conf-managing.conf + backup: yes + +- name: Disable nm-cloud-setup if present + when: + - ansible_distribution_major_version | int > 7 + - ansible_os_family == 'RedHat' + block: + - name: Disable nm-cloud-setup if present + ignore_errors: yes + loop_control: + loop_var: __nm_cloud_setup_disable_item + loop: + - systemctl disable nm-cloud-setup.service nm-cloud-setup.timer + - systemctl stop nm-cloud-setup.service nm-cloud-setup.timer + - ip rule del prio 30400 + - rm -rf /etc/systemd/system/nm-cloud-setup.service.d + ansible.builtin.command: "{{ __nm_cloud_setup_disable_item }}" + + - name: Ensure NetworkManager is running to maintain DHCP + ansible.builtin.service: + name: NetworkManager + state: restarted diff --git a/roles/infrastructure/krb5_client/tasks/freeipa_dbus_patch.yml b/roles/infrastructure/krb5_client/tasks/freeipa_dbus_patch.yml new file mode 100644 index 00000000..e5602af8 --- /dev/null +++ b/roles/infrastructure/krb5_client/tasks/freeipa_dbus_patch.yml @@ -0,0 +1,29 @@ +--- +- name: Copy DBUS config file + copy: + src: dbus_session.conf + dest: /etc/dbus-1/session-local.conf + ignore_errors: true + register: dbus_config_update + +- name: restart services when DBUS is reconfigured + when: dbus_config_update.changed + block: + - name: Ensure dbus is enabled and unmasked + systemd: + name: dbus + enabled: yes + masked: no + ignore_errors: true + + - name: Restart DBUS + service: + name: dbus + state: restarted + ignore_errors: true + + - name: Restart logind + service: + name: systemd-logind + state: restarted + ignore_errors: true diff --git a/roles/infrastructure/krb5_client/tasks/pvc_configs.yml b/roles/infrastructure/krb5_client/tasks/pvc_configs.yml new file mode 100644 index 00000000..205fcb38 --- /dev/null +++ b/roles/infrastructure/krb5_client/tasks/pvc_configs.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Add Renewable ticket lifetime + blockinfile: + dest: "/etc/krb5.conf" + insertafter: 'ticket_lifetime = 24h' + block: | + renew_lifetime = 7d + max_life = 365d + max_renewable_life = 365d + ignore_errors: true + +- name: Comment default_ccache_name in krb5.conf + replace: + dest: /etc/krb5.conf + regexp: 'default_ccache_name = KEYRING:persistent:%{uid}' + replace: '#default_ccache_name = KEYRING:persistent:%{uid}' + ignore_errors: true + +- name: Adding enctypes for Hue + blockinfile: + dest: "/etc/krb5.conf" + insertafter: 'ticket_lifetime = 24h' + block: | + default_tgs_enctypes= des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 + default_tkt_enctypes = des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 + ignore_errors: true \ No newline at end of file diff --git a/roles/infrastructure/krb5_common/defaults/main.yml b/roles/infrastructure/krb5_common/defaults/main.yml index de3b9389..c8378fbf 100644 --- a/roles/infrastructure/krb5_common/defaults/main.yml +++ b/roles/infrastructure/krb5_common/defaults/main.yml @@ -14,7 +14,26 @@ krb5_realm: CLOUDERA.LOCAL krb5_domain: "{{ krb5_realm | lower }}" -krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" -krb5_kdc_admin_password: changeme -krb5_kdc_type: MIT KDC -krb5_enc_types: "aes256-cts aes128-cts" \ No newline at end of file +krb5_kdc_admin_user: "{% if freeipa_activated == false %}cloudera-scm/{% endif %}admin@{{ krb5_realm }}" +krb5_kdc_admin_password: "{{ cloudera_manager_admin_password }}" +krb5_kdc_type: "{{ 'MIT KDC' if (freeipa_activated | default(false)) == false else 'Red Hat IPA' }}" +krb5_enc_types: "aes256-cts aes128-cts" + +skip_krb5_conf_distribution: "{{ freeipa_activated | default(False) }}" +ipa_ldap_url: "ldaps://{{ groups['krb5_server'][0] | default('localhost') }}:636" + +ipa_directory_manager: "cn=Directory Manager" +ipadm_password: "{{ cloudera_manager_admin_password }}" + +ipa_admin_user: admin +ipaadmin_password: "{{ cloudera_manager_admin_password }}" + +ipa_admins_group: admins + +ipa_ldap_dc_suffix: "{% for i in krb5_realm.split('.') %}dc={{ i | lower }}{% if not loop.last %},{% endif %}{% endfor %}" +ipa_ldap_user_bind_dn: "uid=admin,cn=users,cn=accounts,{{ ipa_ldap_dc_suffix }}" +ipa_ldap_user_bind_password: "{{ cloudera_manager_admin_password }}" +ipa_ldap_user_search_base: "cn=users,cn=accounts,{{ ipa_ldap_dc_suffix }}" +ipa_ldap_user_search_filter: "(&(uid={0})(objectClass=person))" +ipa_ldap_group_search_base: "cn=groups,cn=accounts,{{ ipa_ldap_dc_suffix }}" +ipa_ldap_user_group_filter: "(&(member={1})(objectClass=posixgroup))" diff --git a/roles/infrastructure/krb5_server/defaults/main.yml b/roles/infrastructure/krb5_server/defaults/main.yml index eb74de38..561cc676 100644 --- a/roles/infrastructure/krb5_server/defaults/main.yml +++ b/roles/infrastructure/krb5_server/defaults/main.yml @@ -13,4 +13,4 @@ # limitations under the License. --- -krb5_kdc_master_password: changeme +krb5_kdc_master_password: "{{ cloudera_manager_admin_password }}" diff --git a/roles/infrastructure/krb5_server/tasks/fix_freeipa_collection.yml b/roles/infrastructure/krb5_server/tasks/fix_freeipa_collection.yml new file mode 100644 index 00000000..4eb895df --- /dev/null +++ b/roles/infrastructure/krb5_server/tasks/fix_freeipa_collection.yml @@ -0,0 +1,51 @@ +--- +- name: gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - "vars/{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "vars/{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "vars/{{ ansible_facts['distribution'] }}.yml" + - "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "vars/{{ ansible_facts['os_family'] }}.yml" + - "vars/default.yml" + +- name: Ensure NSS is in latest version + package: + name: nss + state: latest + +- name: Install base ipa server packages + package: + name: "{{ ipaserver_packages }}" + state: present + +- name: Add a sleep before calling certmonger for Py27 + lineinfile: + path: /usr/lib/python2.7/site-packages/ipaserver/install/dogtaginstance.py + insertbefore: '.*find_ca_by_nickname.*' + line: ' time.sleep(20)' + state: present + ignore_errors: true + +- name: Raise timeout for CA wait for Py27 + lineinfile: + path: /usr/lib/python2.7/site-packages/ipalib/constants.py + regexp: '^CA_DBUS_TIMEOUT' + line: CA_DBUS_TIMEOUT = 360 + ignore_errors: true + +- name: Add a sleep before calling certmonger for Py36 + lineinfile: + path: /usr/lib/python3.6/site-packages/ipaserver/install/dogtaginstance.py + insertbefore: '.*find_ca_by_nickname.*' + line: ' time.sleep(20)' + state: present + ignore_errors: true + +- name: Raise timeout for CA wait for Py36 + lineinfile: + path: /usr/lib/python3.6/site-packages/ipalib/constants.py + regexp: '^CA_DBUS_TIMEOUT' + line: CA_DBUS_TIMEOUT = 360 + ignore_errors: true diff --git a/roles/infrastructure/krb5_server/tasks/freeipa.yml b/roles/infrastructure/krb5_server/tasks/freeipa.yml index d79adf99..b7508067 100644 --- a/roles/infrastructure/krb5_server/tasks/freeipa.yml +++ b/roles/infrastructure/krb5_server/tasks/freeipa.yml @@ -13,11 +13,94 @@ # limitations under the License. --- +- name: Run FreeIPA Collection fix before IPA Server deployment + include_tasks: + file: fix_freeipa_collection.yml + +- name: Disable SELinux to allow FreeIPA server setup on Rhel8 + when: + - ansible_distribution_major_version | int >= 8 + selinux: + policy: targeted + state: permissive + ignore_errors: yes + - name: Setup FreeIPA Server ansible.builtin.include_role: name: freeipa.ansible_freeipa.ipaserver vars: state: present - ipaserver_realm: "{{ krb5_realm }}" - ipaserver_domain: "{{ krb5_domain }}" + ipaserver_realm: "{{ krb5_realm | upper }}" + ipaserver_domain: "{{ krb5_domain | default(krb5_realm | lower) }}" ipaserver_setup_firewalld: "no" + ipaserver_setup_dns: "{{ freeipa_autodns | default(omit) }}" + ipaserver_auto_forwarders: "{{ freeipa_autodns | default(omit) }}" + +- name: Ensure FreeIPA Superuser if required + when: + - freeipa_superuser is defined + - freeipa_superuser | length > 0 + block: + - name: Create Superuser if not present + community.general.ipa_user: + name: "{{ freeipa_superuser }}" + givenname: "{{ freeipa_superuser_gn | default('Cloudera') }}" + sn: "{{ freeipa_superuser_sn | default('Labs') }}" + password: "{{ freeipa_superuser_pw | default(cloudera_manager_admin_password) }}" + update_password: on_create + ipa_host: "{{ groups.krb5_server | first }}" + ipa_pass: "{{ ipaadmin_password }}" + ipa_user: "{{ ipa_admin_user }}" + + - name: Ensure Superuser is added to admins group + community.general.ipa_group: + name: "{{ ipa_admins_group }}" + user: + - "{{ freeipa_superuser }}" + append: true + ipa_host: "{{ groups.krb5_server | first }}" + ipa_pass: "{{ ipaadmin_password }}" + ipa_user: "{{ ipa_admin_user }}" + +- name: Create FreeIPA DNS records for PVC ECS + when: + - pvc_type is defined and freeipa_autodns is defined + - pvc_type == 'ECS' | default(false) + - freeipa_autodns | default(false) + block: + - name: Gather facts from ECS Server + ansible.builtin.setup: + gather_subset: + - 'default_ipv4' + delegate_to: "{{ ecs_ip_collect_item }}" + delegate_facts: true + loop: "{{ groups['ecs_ecs_server'] }}" + loop_control: + loop_var: ecs_ip_collect_item + + - set_fact: + ecs_server_ip: "{{ hostvars[groups.ecs_ecs_server | first].ansible_facts.default_ipv4.address }}" + + - name: Ensure apps zone is present for PvC ECS + community.general.ipa_dnszone: + ipa_host: "{{ groups.krb5_server | first }}" + ipa_pass: "{{ ipaadmin_password }}" + ipa_user: "{{ ipa_admin_user }}" + state: present + zone_name: "apps.{{ krb5_realm | lower }}" + + - name: Ensure wildcard record is prepared for PvC ECS + community.general.ipa_dnsrecord: + ipa_host: "{{ groups.krb5_server | first }}" + ipa_pass: "{{ ipaadmin_password }}" + ipa_user: "{{ ipa_admin_user }}" + state: present + zone_name: "{{ __dns_record_item }}" + record_name: "*" + record_type: A + record_value: "{{ ecs_server_ip }}" + loop: + - "apps.{{ krb5_realm | lower }}" + - "{{ krb5_realm | lower }}" + loop_control: + loop_var: __dns_record_item diff --git a/roles/infrastructure/krb5_server/vars/RedHat-7.yml b/roles/infrastructure/krb5_server/vars/RedHat-7.yml new file mode 100644 index 00000000..c2f9ba98 --- /dev/null +++ b/roles/infrastructure/krb5_server/vars/RedHat-7.yml @@ -0,0 +1,2 @@ +--- +ipaserver_packages: [ "ipa-server", "libselinux-python" ] \ No newline at end of file diff --git a/roles/infrastructure/krb5_server/vars/RedHat-8.yml b/roles/infrastructure/krb5_server/vars/RedHat-8.yml new file mode 100644 index 00000000..d49fa517 --- /dev/null +++ b/roles/infrastructure/krb5_server/vars/RedHat-8.yml @@ -0,0 +1,2 @@ +--- +ipaserver_packages: [ "@idm:DL1/server" ] \ No newline at end of file diff --git a/roles/infrastructure/krb5_server/vars/Ubuntu.yml b/roles/infrastructure/krb5_server/vars/Ubuntu.yml new file mode 100644 index 00000000..ac96c558 --- /dev/null +++ b/roles/infrastructure/krb5_server/vars/Ubuntu.yml @@ -0,0 +1,2 @@ +--- +ipaserver_packages: [ "freeipa-server" ] \ No newline at end of file diff --git a/roles/infrastructure/krb5_server/vars/default.yml b/roles/infrastructure/krb5_server/vars/default.yml new file mode 100644 index 00000000..6324f7ec --- /dev/null +++ b/roles/infrastructure/krb5_server/vars/default.yml @@ -0,0 +1,2 @@ +--- +ipaserver_packages: [ "ipa-server", "python3-libselinux" ] diff --git a/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml b/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml index e10a1d58..793c333e 100644 --- a/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml +++ b/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml @@ -76,9 +76,6 @@ register: _ranger_https_resp changed_when: false -- set_fact: - _ranger_https_used: "{{ _ranger_https_resp.status == 401 }}" - - name: Check if HTTP is used uri: url: "http://{{ _ranger_host }}:{{ _ranger_http_port }}" @@ -87,10 +84,10 @@ - -1 register: _ranger_http_resp changed_when: false - when: not _ranger_https_used - set_fact: - _ranger_http_used: "{{ not _ranger_https_used and _ranger_http_used.status | default(omit) == 401 }}" + _ranger_https_used: "{{ ( _ranger_https_resp.status == 401 ) | bool }}" + _ranger_http_used: "{{ ( _ranger_http_resp.status == 401 ) | bool }}" - fail: msg: "Unable to connect to the Ranger admin on {{ _ranger_host }}." @@ -98,13 +95,13 @@ - not _ranger_https_used - not _ranger_http_used -- set_fact: - ranger_url: "https://{{ _ranger_host }}:{{ _ranger_https_port }}" - when: _ranger_https_used - - set_fact: ranger_url: "http://{{ _ranger_host }}:{{ _ranger_http_port }}" when: _ranger_http_used +- set_fact: + ranger_url: "https://{{ _ranger_host }}:{{ _ranger_https_port }}" + when: _ranger_https_used + - set_fact: ranger_api_url: "{{ ranger_url }}/service/public/v2/api" diff --git a/roles/operations/restart_cluster/meta/main.yml b/roles/operations/restart_cluster/meta/main.yml new file mode 100644 index 00000000..524838cc --- /dev/null +++ b/roles/operations/restart_cluster/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/operations/restart_cluster/tasks/main.yml b/roles/operations/restart_cluster/tasks/main.yml new file mode 100644 index 00000000..c6077a57 --- /dev/null +++ b/roles/operations/restart_cluster/tasks/main.yml @@ -0,0 +1,13 @@ +--- + +- name: Restart cluster + cm_api: + endpoint: /clusters/{{ cluster_to_restart }}/commands/restart + method: POST + timeout: "{{ cluster_restart_timeout | default(3000) }}" + +- name: Re-deploy client configurations + cm_api: + endpoint: /clusters/{{ cluster_to_restart }}/commands/deployClientConfig + method: POST + timeout: "{{ cluster_restart_timeout | default(3000) }}" diff --git a/roles/operations/restart_cluster_services/meta/main.yml b/roles/operations/restart_cluster_services/meta/main.yml new file mode 100644 index 00000000..3a5a0054 --- /dev/null +++ b/roles/operations/restart_cluster_services/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/operations/restart_cluster_services/tasks/main.yml b/roles/operations/restart_cluster_services/tasks/main.yml new file mode 100644 index 00000000..72f8b8bf --- /dev/null +++ b/roles/operations/restart_cluster_services/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Get All services from CM + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_name | urlencode() }}/services" + register: cloudera_manager_all_services + +- name: Handle Restarts + include_tasks: service_restart.yml + loop: "{{ services_to_restart }}" + loop_control: + loop_var: __service_restart_item \ No newline at end of file diff --git a/roles/operations/restart_cluster_services/tasks/service_restart.yml b/roles/operations/restart_cluster_services/tasks/service_restart.yml new file mode 100644 index 00000000..ba174c19 --- /dev/null +++ b/roles/operations/restart_cluster_services/tasks/service_restart.yml @@ -0,0 +1,15 @@ +--- +- name: Get Specific Service Name + set_fact: + restart_service_name: "{{ cloudera_manager_all_services | community.general.json_query(query) }}" + vars: + query: "items[?type == '{{ __service_restart_item }}'].name | [0]" + +- name: Restart Cluster Service + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ cluster_base_name | urlencode() }}/services/{{ restart_service_name | lower }}/commands/restart" + method: POST + +- name: Wait for restart to acknowledge + wait_for: + timeout: 15 \ No newline at end of file diff --git a/roles/operations/restart_mgmt_services/meta/main.yml b/roles/operations/restart_mgmt_services/meta/main.yml new file mode 100644 index 00000000..0d623502 --- /dev/null +++ b/roles/operations/restart_mgmt_services/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.services_info diff --git a/roles/operations/restart_mgmt_services/tasks/main.yml b/roles/operations/restart_mgmt_services/tasks/main.yml new file mode 100644 index 00000000..3d02e424 --- /dev/null +++ b/roles/operations/restart_mgmt_services/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Restart Cloudera Manager Management Services + cloudera.cluster.cm_api: + endpoint: "/cm/service/roleCommands/restart" + method: POST + body: + items: "{{ services_to_restart }}" \ No newline at end of file diff --git a/roles/operations/stop_cluster/meta/main.yml b/roles/operations/stop_cluster/meta/main.yml new file mode 100644 index 00000000..69f6a359 --- /dev/null +++ b/roles/operations/stop_cluster/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera_manager/api_client diff --git a/roles/operations/stop_cluster/tasks/main.yml b/roles/operations/stop_cluster/tasks/main.yml new file mode 100644 index 00000000..18da9ec3 --- /dev/null +++ b/roles/operations/stop_cluster/tasks/main.yml @@ -0,0 +1,51 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Check the cluster exists + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster.name | urlencode() }} + method: GET + status_code: 200,404 + changed_when: false + register: cluster_before_stop + +- name: Stop the cluster + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster.name | urlencode() }}/commands/stop + method: POST + register: services_stop + when: + - cluster_before_stop.status == 200 + failed_when: services_stop.status | default(0) != 200 + +- name: Wait for the cluster services to stop + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster.name | urlencode() }}/services + method: GET + status_code: 200,404 + changed_when: false + register: services + when: + - cluster_before_stop.status == 200 + until: >- + 'json' in services + and services.json | json_query(stopped_query) | length + == services.json | json_query(all_query) | length + vars: + stopped_query: 'items[?(serviceState==`STOPPED` || serviceState==`NA`)]' + all_query: 'items[*]' + retries: "{{ teardown_stop_cluster_poll_max_retries | default(30) }}" + delay: "{{ teardown_stop_cluster_poll_duration | default(20) }}" diff --git a/roles/prereqs/mysql_connector/tasks/main.yml b/roles/prereqs/mysql_connector/tasks/main.yml index 8e02d640..3fa9d14a 100644 --- a/roles/prereqs/mysql_connector/tasks/main.yml +++ b/roles/prereqs/mysql_connector/tasks/main.yml @@ -49,3 +49,32 @@ dest: /usr/share/java/mysql-connector-java.jar remote_src: yes mode: 0644 + +# MySql on rhel8 fix +- name: Fix MySQL on RHEL8 + when: + - ansible_distribution == "RedHat" + - ansible_distribution_major_version == "8" + block: + - name: Remove mariadb packages if needed + package: + name: mariadb-devel + state: absent + + - name: Install mysql-devel package + package: + name: mysql-devel + state: installed + + - name: Send my_config.h for RHEL 8 + template: + src: my_config.h + dest: /usr/include/mysql/my_config.h + + - name: Install Mysql packages for python + shell: pip install MySQL-python --force-reinstall --ignore-installed + ignore_errors: true + + - name: Chmod on mysql python files + shell: chmod -R 755 /usr/lib64/python2.7/site-packages/MySQLdb /usr/lib64/python2.7/site-packages/MySQL_python* /usr/lib64/python2.7/site-packages/_mysql* + ignore_errors: true \ No newline at end of file diff --git a/roles/prereqs/mysql_connector/templates/my_config.h b/roles/prereqs/mysql_connector/templates/my_config.h new file mode 100644 index 00000000..f5efd7b1 --- /dev/null +++ b/roles/prereqs/mysql_connector/templates/my_config.h @@ -0,0 +1,355 @@ +/* Copyright (c) 2009, 2021, Oracle and/or its affiliates. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + #ifndef MY_CONFIG_H + #define MY_CONFIG_H + + /* + * From configure.cmake, in order of appearance + */ + + /* Libraries */ + #define HAVE_LIBM 1 + /* #undef HAVE_LIBNSL */ + #define HAVE_LIBCRYPT 1 + /* #undef HAVE_LIBSOCKET */ + #define HAVE_LIBDL 1 + #define HAVE_LIBRT 1 + /* #undef HAVE_LIBWRAP */ + /* #undef HAVE_LIBWRAP_PROTOTYPES */ + + /* Header files */ + #define HAVE_ALLOCA_H 1 + #define HAVE_ARPA_INET_H 1 + #define HAVE_DLFCN_H 1 + #define HAVE_EXECINFO_H 1 + #define HAVE_FPU_CONTROL_H 1 + #define HAVE_GRP_H 1 + #define HAVE_LANGINFO_H 1 + #define HAVE_MALLOC_H 1 + #define HAVE_NETINET_IN_H 1 + #define HAVE_POLL_H 1 + #define HAVE_PWD_H 1 + #define HAVE_STRINGS_H 1 + #define HAVE_SYS_CDEFS_H 1 + #define HAVE_SYS_IOCTL_H 1 + #define HAVE_SYS_MMAN_H 1 + #define HAVE_SYS_PRCTL_H 1 + #define HAVE_SYS_RESOURCE_H 1 + #define HAVE_SYS_SELECT_H 1 + #define HAVE_SYS_SOCKET_H 1 + #define HAVE_TERM_H 1 + #define HAVE_TERMIOS_H 1 + #define HAVE_TERMIO_H 1 + #define HAVE_UNISTD_H 1 + #define HAVE_SYS_WAIT_H 1 + #define HAVE_SYS_PARAM_H 1 + #define HAVE_FNMATCH_H 1 + #define HAVE_SYS_UN_H 1 + #define HAVE_SASL_SASL_H 1 + + /* Functions */ + /* #undef HAVE_ALIGNED_MALLOC */ + #define HAVE_BACKTRACE 1 + #define HAVE_INDEX 1 + #define HAVE_CHOWN 1 + #define HAVE_CUSERID 1 + /* #undef HAVE_DIRECTIO */ + #define HAVE_FTRUNCATE 1 + #define HAVE_FCHMOD 1 + #define HAVE_FCNTL 1 + #define HAVE_FDATASYNC 1 + #define HAVE_DECL_FDATASYNC 1 + #define HAVE_FEDISABLEEXCEPT 1 + #define HAVE_FSYNC 1 + /* #undef HAVE_GETHRTIME */ + #define HAVE_GETPASS 1 + /* #undef HAVE_GETPASSPHRASE */ + #define HAVE_GETPWNAM 1 + #define HAVE_GETPWUID 1 + #define HAVE_GETRUSAGE 1 + #define HAVE_INITGROUPS 1 + /* #undef HAVE_ISSETUGID */ + #define HAVE_GETUID 1 + #define HAVE_GETEUID 1 + #define HAVE_GETGID 1 + #define HAVE_GETEGID 1 + /* #undef HAVE_LSAN_DO_RECOVERABLE_LEAK_CHECK */ + #define HAVE_MADVISE 1 + #define HAVE_MALLOC_INFO 1 + #define HAVE_MLOCK 1 + #define HAVE_MLOCKALL 1 + #define HAVE_MMAP64 1 + #define HAVE_POLL 1 + #define HAVE_POSIX_FALLOCATE 1 + #define HAVE_POSIX_MEMALIGN 1 + #define HAVE_PTHREAD_CONDATTR_SETCLOCK 1 + #define HAVE_PTHREAD_GETAFFINITY_NP 1 + #define HAVE_PTHREAD_SIGMASK 1 + #define HAVE_SLEEP 1 + #define HAVE_STPCPY 1 + #define HAVE_STPNCPY 1 + /* #undef HAVE_STRLCPY */ + /* #undef HAVE_STRLCAT */ + #define HAVE_STRSIGNAL 1 + /* #undef HAVE_TELL */ + #define HAVE_VASPRINTF 1 + #define HAVE_MEMALIGN 1 + #define HAVE_NL_LANGINFO 1 + /* #undef HAVE_HTONLL */ + #define HAVE_EPOLL 1 + + /* WL2373 */ + #define HAVE_SYS_TIME_H 1 + #define HAVE_SYS_TIMES_H 1 + #define HAVE_TIMES 1 + #define HAVE_GETTIMEOFDAY 1 + + /* Symbols */ + #define HAVE_LRAND48 1 + #define GWINSZ_IN_SYS_IOCTL 1 + #define FIONREAD_IN_SYS_IOCTL 1 + /* #undef FIONREAD_IN_SYS_FILIO */ + #define HAVE_MADV_DONTDUMP 1 + #define HAVE_O_TMPFILE + + /* #undef HAVE_KQUEUE */ + #define HAVE_SETNS 1 + /* #undef HAVE_KQUEUE_TIMERS */ + #define HAVE_POSIX_TIMERS 1 + + /* Endianess */ + /* #undef WORDS_BIGENDIAN */ + #define HAVE_ENDIAN_CONVERSION_MACROS 1 + + /* Type sizes */ + #define SIZEOF_VOIDP 8 + #define SIZEOF_CHARP 8 + #define SIZEOF_LONG 8 + #define SIZEOF_SHORT 2 + #define SIZEOF_INT 4 + #define SIZEOF_LONG_LONG 8 + #define SIZEOF_TIME_T 8 + #define HAVE_ULONG 1 + #define HAVE_U_INT32_T 1 + #define HAVE_TM_GMTOFF 1 + + /* Support for tagging symbols with __attribute__((visibility("hidden"))) */ + #define HAVE_VISIBILITY_HIDDEN 1 + + /* Code tests*/ + #define HAVE_CLOCK_GETTIME 1 + #define HAVE_CLOCK_REALTIME 1 + #define STACK_DIRECTION -1 + #define TIME_WITH_SYS_TIME 1 + /* #undef NO_FCNTL_NONBLOCK */ + #define HAVE_PAUSE_INSTRUCTION 1 + /* #undef HAVE_FAKE_PAUSE_INSTRUCTION */ + /* #undef HAVE_HMT_PRIORITY_INSTRUCTION */ + #define HAVE_ABI_CXA_DEMANGLE 1 + #define HAVE_BUILTIN_UNREACHABLE 1 + #define HAVE_BUILTIN_EXPECT 1 + #define HAVE_BUILTIN_STPCPY 1 + #define HAVE_GCC_SYNC_BUILTINS 1 + /* #undef HAVE_VALGRIND */ + #define HAVE_SYS_GETTID 1 + /* #undef HAVE_PTHREAD_GETTHREADID_NP */ + /* #undef HAVE_PTHREAD_THREADID_NP */ + #define HAVE_INTEGER_PTHREAD_SELF 1 + #define HAVE_PTHREAD_SETNAME_NP 1 + + /* IPV6 */ + /* #undef HAVE_NETINET_IN6_H */ + /* #undef HAVE_STRUCT_IN6_ADDR */ + + /* + * Platform specific CMake files + */ + #define MACHINE_TYPE "x86_64" + /* #undef LINUX_ALPINE */ + /* #undef LINUX_SUSE */ + #define HAVE_LINUX_LARGE_PAGES 1 + /* #undef HAVE_SOLARIS_LARGE_PAGES */ + /* #undef HAVE_SOLARIS_ATOMIC */ + /* #undef WITH_SYSTEMD_DEBUG */ + #define SYSTEM_TYPE "Linux" + /* This should mean case insensitive file system */ + /* #undef FN_NO_CASE_SENSE */ + + /* + * From main CMakeLists.txt + */ + /* #undef CHECK_ERRMSG_FORMAT */ + #define MAX_INDEXES 64U + /* #undef WITH_INNODB_MEMCACHED */ + /* #undef ENABLE_MEMCACHED_SASL */ + /* #undef ENABLE_MEMCACHED_SASL_PWDB */ + #define ENABLED_PROFILING 1 + /* #undef HAVE_ASAN */ + /* #undef HAVE_LSAN */ + /* #undef HAVE_UBSAN */ + /* #undef HAVE_TSAN */ + /* #undef ENABLED_LOCAL_INFILE */ + /* #undef KERBEROS_LIB_CONFIGURED */ + #define SCRAM_LIB_CONFIGURED + /* #undef WITH_HYPERGRAPH_OPTIMIZER */ + + /* Lock Order */ + /* #undef WITH_LOCK_ORDER */ + + /* Character sets and collations */ + #define DEFAULT_MYSQL_HOME "/usr/local/mysql" + #define SHAREDIR "/usr/local/mysql/share" + #define DEFAULT_BASEDIR "/usr/local/mysql" + #define MYSQL_DATADIR "/usr/local/mysql/data" + #define MYSQL_KEYRINGDIR "/usr/local/mysql/keyring" + #define DEFAULT_CHARSET_HOME "/usr/local/mysql" + #define PLUGINDIR "/usr/local/mysql/lib/plugin" + #define DEFAULT_SYSCONFDIR "/usr/local/mysql/etc" + #define DEFAULT_TMPDIR P_tmpdir + /* + * Readline + */ + #define HAVE_MBSTATE_T + #define HAVE_LANGINFO_CODESET + #define HAVE_WCSDUP + #define HAVE_WCHAR_T 1 + #define HAVE_WINT_T 1 + #define HAVE_CURSES_H 1 + /* #undef HAVE_NCURSES_H */ + #define USE_LIBEDIT_INTERFACE 1 + #define HAVE_HIST_ENTRY 1 + #define USE_NEW_EDITLINE_INTERFACE 1 + #define EDITLINE_HAVE_COMPLETION_CHAR 1 + /* #undef EDITLINE_HAVE_COMPLETION_INT */ + + + /* + * Libedit + */ + #define HAVE_GETLINE 1 + /* #undef HAVE___SECURE_GETENV */ + #define HAVE_SECURE_GETENV 1 + /* #undef HAVE_VIS */ + /* #undef HAVE_UNVIS */ + /* #undef HAVE_GETPW_R_DRAFT */ + #define HAVE_GETPW_R_POSIX + + /* + * Character sets + */ + #define MYSQL_DEFAULT_CHARSET_NAME "utf8mb4" + #define MYSQL_DEFAULT_COLLATION_NAME "utf8mb4_0900_ai_ci" + + /* + * Performance schema + */ + #define WITH_PERFSCHEMA_STORAGE_ENGINE 1 + /* #undef DISABLE_PSI_THREAD */ + /* #undef DISABLE_PSI_MUTEX */ + /* #undef DISABLE_PSI_RWLOCK */ + /* #undef DISABLE_PSI_COND */ + /* #undef DISABLE_PSI_FILE */ + /* #undef DISABLE_PSI_TABLE */ + /* #undef DISABLE_PSI_SOCKET */ + /* #undef DISABLE_PSI_STAGE */ + /* #undef DISABLE_PSI_STATEMENT */ + /* #undef DISABLE_PSI_SP */ + /* #undef DISABLE_PSI_PS */ + /* #undef DISABLE_PSI_IDLE */ + /* #undef DISABLE_PSI_ERROR */ + /* #undef DISABLE_PSI_STATEMENT_DIGEST */ + /* #undef DISABLE_PSI_METADATA */ + /* #undef DISABLE_PSI_MEMORY */ + /* #undef DISABLE_PSI_TRANSACTION */ + + /* + * MySQL version + */ + #define MYSQL_VERSION_MAJOR 8 + #define MYSQL_VERSION_MINOR 0 + #define MYSQL_VERSION_PATCH 25 + #define MYSQL_VERSION_EXTRA "" + #define PACKAGE "mysql" + #define PACKAGE_VERSION "8.0.25" + #define VERSION "8.0.25" + #define PROTOCOL_VERSION 10 + + /* + * CPU info + */ + #define CPU_LEVEL1_DCACHE_LINESIZE 64 + + + /* + * NDB + */ + /* #undef HAVE_GETRLIMIT */ + /* #undef WITH_NDBCLUSTER_STORAGE_ENGINE */ + /* #undef HAVE_PTHREAD_SETSCHEDPARAM */ + + /* + * Other + */ + /* #undef EXTRA_DEBUG */ + #define HANDLE_FATAL_SIGNALS 1 + + /* + * Hardcoded values needed by libevent/NDB/memcached + */ + #define HAVE_FCNTL_H 1 + #define HAVE_GETADDRINFO 1 + #define HAVE_INTTYPES_H 1 + /* libevent's select.c is not Windows compatible */ + #ifndef _WIN32 + #define HAVE_SELECT 1 + #endif + #define HAVE_SIGNAL_H 1 + #define HAVE_STDARG_H 1 + #define HAVE_STDINT_H 1 + #define HAVE_STDLIB_H 1 + #define HAVE_STRTOK_R 1 + #define HAVE_STRTOLL 1 + #define HAVE_SYS_STAT_H 1 + #define HAVE_SYS_TYPES_H 1 + #define SIZEOF_CHAR 1 + + /* For --secure-file-priv */ + #define DEFAULT_SECURE_FILE_PRIV_DIR "NULL" + #define HAVE_LIBNUMA 1 + + /* For default value of --early_plugin_load */ + /* #undef DEFAULT_EARLY_PLUGIN_LOAD */ + + /* For default value of --partial_revokes */ + #define DEFAULT_PARTIAL_REVOKES 0 + + #define SO_EXT ".so" + + + /* From libmysql/CMakeLists.txt */ + #define HAVE_UNIX_DNS_SRV 1 + /* #undef HAVE_WIN32_DNS_SRV */ + + /* ARM crc32 support */ + /* #undef HAVE_ARMV8_CRC32_INTRINSIC */ + + #endif \ No newline at end of file diff --git a/roles/prereqs/os/defaults/main.yml b/roles/prereqs/os/defaults/main.yml index 1573273c..8d159343 100644 --- a/roles/prereqs/os/defaults/main.yml +++ b/roles/prereqs/os/defaults/main.yml @@ -28,5 +28,6 @@ unnecessary_services: - ip6tables - postfix - tuned + - firewalld # Added for ECS deployments on RedHat selinux_state: permissive diff --git a/roles/prereqs/os/tasks/main-RedHat.yml b/roles/prereqs/os/tasks/main-RedHat.yml index 1219f615..e770815b 100644 --- a/roles/prereqs/os/tasks/main-RedHat.yml +++ b/roles/prereqs/os/tasks/main-RedHat.yml @@ -13,7 +13,7 @@ # limitations under the License. --- -- name: Setup System python on Rhel8 +- name: Setup System python3 on Rhel8 when: ansible_distribution_major_version | int >= 8 block: - name: Check if Python3 is installed so we don't end up with multiple versions @@ -30,6 +30,14 @@ update_cache: yes state: present + - name: Ensure pip3 is upgraded + ansible.builtin.command: "pip3 install --upgrade pip" + +# leaving as separate group for when py2 is finally deprecated +- name: Setup System python2 on Rhel8 + when: + - ansible_distribution_major_version | int >= 8 + block: - name: Check if Python2 is installed so we don't end up with multiple versions shell: python2 --version register: __py2_check @@ -50,9 +58,6 @@ alternatives --set python /usr/bin/python2 fi - - name: Ensure pip3 is upgraded - ansible.builtin.command: "pip3 install --upgrade pip" - - name: Disable SELinux selinux: policy: targeted diff --git a/roles/prereqs/os/tasks/main.yml b/roles/prereqs/os/tasks/main.yml index f0e8e940..e6d1a05a 100644 --- a/roles/prereqs/os/tasks/main.yml +++ b/roles/prereqs/os/tasks/main.yml @@ -36,13 +36,15 @@ when: not((ansible_virtualization_type == "docker" or ansible_virtualization_type == "container") and ansible_virtualization_role == "guest") - name: Populate service facts - service_facts: + ansible.builtin.service_facts: - name: Disable unnecessary services - service: + register: __disable_services + ansible.builtin.service: name: "{{ item }}" state: stopped enabled: no + ignore_errors: yes # fails sometimes with systemd on centos7 where ip6tables are not present when: "item + '.service' in ansible_facts.services" with_items: "{{ unnecessary_services }}" diff --git a/roles/prereqs/postgresql_connector/defaults/main.yml b/roles/prereqs/postgresql_connector/defaults/main.yml index f6baf898..9ceced43 100644 --- a/roles/prereqs/postgresql_connector/defaults/main.yml +++ b/roles/prereqs/postgresql_connector/defaults/main.yml @@ -16,3 +16,4 @@ postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.3.3.jar postgresql_connector_checksum: md5:bef0b2e1c6edcd8647c24bed31e1a4ac +install_py3_psycopg2: false \ No newline at end of file diff --git a/roles/prereqs/postgresql_connector/tasks/main.yml b/roles/prereqs/postgresql_connector/tasks/main.yml index 3ec10f16..fcf2564b 100644 --- a/roles/prereqs/postgresql_connector/tasks/main.yml +++ b/roles/prereqs/postgresql_connector/tasks/main.yml @@ -35,3 +35,16 @@ src: "{{ local_temp_dir }}/postgresql-connector-java.jar" dest: /usr/share/java/postgresql-connector-java.jar mode: 0644 + +# SSB will need the python3-psycopg2 connector + +- name: Create python3-psycopg2 directory + file: + path: "/usr/share/python3" + state: directory + mode: '777' + when: install_py3_psycopg2 == true + +- name: Install python3-psycopg2 + shell: "pip3 install psycopg2-binary==2.8.5 -t /usr/share/python3" + when: install_py3_psycopg2 == true \ No newline at end of file diff --git a/roles/prereqs/pvc_ecs/tasks/main.yml b/roles/prereqs/pvc_ecs/tasks/main.yml new file mode 100644 index 00000000..da15484f --- /dev/null +++ b/roles/prereqs/pvc_ecs/tasks/main.yml @@ -0,0 +1,79 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install necessary additional packages + ansible.builtin.package: + name: "{{ __package_item }}" + state: latest + loop: + - nfs-utils + loop_control: + loop_var: __package_item + +- name: Setup iptables on rhel7 + when: ansible_distribution_major_version | int == 7 + block: + - name: Install iptables for rhel7 + ansible.builtin.package: + lock_timeout: 180 + name: "{{ __iptables_item }}" + update_cache: yes + state: present + loop: + - iptables + - iptables-services + loop_control: + loop_var: __iptables_item + + - name: start iptables on rhel7 + systemd: + name: iptables + enabled: yes + state: started + daemon-reload: yes + +- name: Setup nftables on rhel8 + when: ansible_distribution_major_version | int >= 8 + block: + - name: Install iptables for rhel8 + ansible.builtin.package: + lock_timeout: 180 + name: "{{ __iptables_install_item }}" + update_cache: yes + state: present + loop: + - iptables-services + loop_control: + loop_var: __iptables_install_item + + - name: start nftables on rhel8 + systemd: + name: iptables + enabled: yes + state: started + daemon-reload: yes + +- name: Flush IPTables + ansible.builtin.iptables: + flush: yes + table: "{{ __iptables_flush_item }}" + loop: + - filter + - nat + - mangle + - raw + - security + loop_control: + loop_var: __iptables_flush_item \ No newline at end of file diff --git a/roles/prereqs/user_accounts_ecs/defaults/main.yml b/roles/prereqs/user_accounts_ecs/defaults/main.yml new file mode 100644 index 00000000..ee37a888 --- /dev/null +++ b/roles/prereqs/user_accounts_ecs/defaults/main.yml @@ -0,0 +1,25 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +skip_user_group_init: false +local_accounts: + + - user: cloudera-scm + home: /var/lib/cloudera-scm-server + comment: Cloudera Manager + mode: '770' + keystore_acl: True + key_acl: True + key_password_acl: True diff --git a/roles/prereqs/user_accounts_ecs/meta/main.yml b/roles/prereqs/user_accounts_ecs/meta/main.yml new file mode 100644 index 00000000..f63fb6e8 --- /dev/null +++ b/roles/prereqs/user_accounts_ecs/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/prereqs/user_accounts_ecs/tasks/main.yml b/roles/prereqs/user_accounts_ecs/tasks/main.yml new file mode 100644 index 00000000..2aa6935b --- /dev/null +++ b/roles/prereqs/user_accounts_ecs/tasks/main.yml @@ -0,0 +1,60 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- block: + + - name: Create hadoop group + group: + name: hadoop + state: present + + - debug: + msg: >- + If you have a failure during user creation, + please ensure that users and groups can be created + locally or that they are already setup correctly, + with no collisions, if managed elsewhere. + The user module error may not identify the root cause. + run_once: true + + - name: Create local user accounts for ECS + ansible.builtin.user: + name: "{{ account.user }}" + home: "{{ account.home }}" + comment: "{{ account.comment | default(omit) }}" + groups: "{{ account.extra_groups | default([]) }}" + uid: "{{ account.uid | default(omit) }}" + shell: "{{ account.shell | default('/sbin/nologin') }}" + append: yes + loop: "{{ local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + when: account.when | default(True) + + - name: Set home directory permissions + file: + path: "{{ account.home }}" + owner: "{{ account.user }}" + group: "{{ account.user }}" + mode: "{{ account.mode | default('0755') }}" + loop: "{{ local_accounts }}" + loop_control: + loop_var: account + when: account.when | default(True) + + when: (not skip_user_group_init) | default(true) + diff --git a/roles/security/tls_install_certs/tasks/main.yml b/roles/security/tls_install_certs/tasks/main.yml index e15ba342..c1ce5a67 100644 --- a/roles/security/tls_install_certs/tasks/main.yml +++ b/roles/security/tls_install_certs/tasks/main.yml @@ -14,12 +14,14 @@ --- -- set_fact: +- name: Set fact for signed TLS certificates directory + ansible.builtin.set_fact: tls_signed_certs_dir: "{{ local_certs_dir }}" when: tls_signed_certs_dir is not defined # remote certificates for ca_server ca -- set_fact: +- name: Define remote certificates for embedded CA server + ansible.builtin.set_fact: tls_ca_certs: - alias: cluster_rootca path: "{{ ca_server_root_cert_path }}" @@ -30,12 +32,22 @@ when: tls_ca_certs is not defined and 'ca_server' in groups # remote certificates for freeipa ca -- set_fact: +- name: Define remote certificates for embedded FreeIPA server + ansible.builtin.set_fact: tls_ca_certs: - alias: cluster_ca path: "/etc/ipa/ca.crt" - remote_host: "{{ groups.krb5_server | first | default(omit) }}" - when: tls_ca_certs is not defined and krb5_kdc_type | default(None) == 'Red Hat IPA' + remote_host: "{{ groups.krb5_server | first }}" + when: tls_ca_certs is not defined and 'krb5_server' in groups and krb5_kdc_type | default(None) == 'Red Hat IPA' + +# remote certificates for freeipa ca +- name: Define remote certificates for sidecar FreeIPA server + ansible.builtin.set_fact: + tls_ca_certs: + - alias: cluster_ca + path: "/etc/ipa/ca.crt" + remote_host: "{{ remote_ipa_server }}" + when: tls_ca_certs is not defined and remote_ipa_server is defined and krb5_kdc_type | default(None) == 'Red Hat IPA' - name: Fetch the remote CA certs fetch: diff --git a/roles/teardown/tasks/main.yml b/roles/teardown/tasks/main.yml index 3b782e6c..bd301f34 100644 --- a/roles/teardown/tasks/main.yml +++ b/roles/teardown/tasks/main.yml @@ -207,7 +207,7 @@ - "'kts_active' in group_names or 'kts_passive' in group_names" - cluster.type | default('base') == 'kts' -# refactor once the KMS locactions are properly configured as overlays +# refactor once the KMS locactions are properly configured as overlays # see teardown_cms.yml for reference and re-purpose teardown_cms_*.yml # https://docs.cloudera.com/cloudera-manager/7.2.0/reference/topics/cm_props_cdh710_keytrusteekms.html - name: Teardown KMS diff --git a/roles/teardown/tasks/teardown_ecs.yml b/roles/teardown/tasks/teardown_ecs.yml new file mode 100644 index 00000000..1c41c31e --- /dev/null +++ b/roles/teardown/tasks/teardown_ecs.yml @@ -0,0 +1,101 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +#- name: Include config cluster defaults for deployment +# ansible.builtin.include_role: +# name: cloudera.cluster.config.cluster.base +# public: yes + +#- name: Include config cluster ecs defaults for deployment +# ansible.builtin.include_role: +# name: cloudera.cluster.config.cluster.ecs +# public: yes + +- name: Ensure properly configured + assert: + that: >- + {{ + (teardown_everything is defined and teardown_everything) + or teardown_cluster is defined + or (teardown_skip_cluster_deletion is defined and teardown_skip_cluster_deletion) + }} + success_msg: "Teardown is properly configured and will execute" + fail_msg: >- + Please set teardown_everything to true to teardown + every cluster and cm. + Alternatively, specify a cluster + to teardown using teardown_cluster ('all' to teardown + every cluster). + You can also set teardown_skip_cluster_deletion to true + to skip cluster deletion in CM. + +- name: Stop ECS CLuster + include_role: + name: cloudera.cluster.operations.stop_cluster + run_once: true + +- name: Run rke2 killall + register: _rke2_killall + shell: /opt/cloudera/parcels/ECS/bin/rke2-killall.sh + failed_when: + - _rke2_killall.failed + - "'No such file or directory' not in _rke2_killall.stderr" + +- name: Run rke2 Uninstall + register: _rke2_uninstall + shell: /opt/cloudera/parcels/ECS/bin/rke2-uninstall.sh; + failed_when: + - _rke2_uninstall.failed + - "'No such file or directory' not in _rke2_uninstall.stderr" + +- name: Delete misc + shell: | + rm -rf /ecs/docker/docker-store/ + rm -rf /ecs/local/* + rm -rf /ecs/longhorn/* + rm -rf /var/lib/docker_server + rm -rf /etc/docker/certs.d + systemctl stop iscsid + yum -y erase iscsi-initiator-utils + rm -rf /var/lib/iscsi + rm -rf /etc/iscsi + rm -rf /etc/cni + rm -f /run/longhorn-iscsi.lock + rm -rf /run/k3s + rm -rf /run/containerd + rm -rf /var/lib/docker/* + rm -rf /var/log/containers/* + rm -rf /var/log/pods/* + ignore_errors: true + + +- name: Flush and Delete IPTables + ansible.builtin.iptables: + flush: yes + table: "{{ __iptables_flush_item }}" + loop: + - filter + - nat + - mangle + - raw + - security + loop_control: + loop_var: __iptables_flush_item + +- name: Delete ECS CLuster + include_role: + name: cloudera.cluster.operations.delete_cluster + run_once: true \ No newline at end of file diff --git a/roles/verify/definition/tasks/main.yml b/roles/verify/definition/tasks/main.yml index 24a3b312..44c7bd5d 100644 --- a/roles/verify/definition/tasks/main.yml +++ b/roles/verify/definition/tasks/main.yml @@ -90,7 +90,7 @@ Unused host template(s) '{{ unmatched_templates_cluster }}' found in cluster definitions - check against inventory. -# Kerberos +# Kerberos - block: - set_fact: expect_kerberos: >- @@ -106,6 +106,7 @@ fail_msg: "The KDC host is configured but no cluster is configured to use Kerberos" ignore_errors: yes when: not expect_kerberos + - name: Ensure that Kerberos is specified when used assert: that: "{{ krb5_kdc_host is defined or 'krb5_server' in groups }}" @@ -114,6 +115,7 @@ The KDC host must be configured, either by adding a host to the 'krb5_server' group or by setting 'krb5_kdc_host' to an existing KDC host when: expect_kerberos + - name: Ensure that the KDC host is only specified in one place assert: that: >- @@ -126,6 +128,7 @@ The var 'krb5_kdc_host' and group 'krb5_server' cannot be used together – please unset one when: expect_kerberos + - name: Ensure that at most one host is specified in the 'krb5_server' group assert: that: "{{ groups['krb5_server'] | length == 1 }}" @@ -135,9 +138,21 @@ host should be specified in the 'krb5_server' group when: "'krb5_server' in groups" -# Service specific + - name: Ensure that FreeIpa and custom_repo are not on the same host + assert: + that: "{{ groups['krb5_server'] != groups['custom_repo'] }}" + success_msg: krb5_server for FreeIPA Server and custom_repo do not share a host + fail_msg: >- + FreeIPA and httpd share a near-hardcoded dependency on port 8443, therefore + they should not be colocated on the same host + when: + - "'krb5_server' in groups" + - "'custom_repo' in groups" + - freeipa_activated == True + +# Service specific -## KMS/KTS +## KMS/KTS - block: - set_fact: kts_clusters: >- @@ -182,7 +197,7 @@ when a kts cluster is present in the cluster definition when: expect_kts -## Ranger +## Ranger - block: - set_fact: kerberos_clusters: >- @@ -260,6 +275,6 @@ fail_msg: "The CM admin password must not be part of the hostname" when: cloudera_manager_admin_password is defined -# Version specific +# Version specific # Add version specific issues here (e.g. Database versions) diff --git a/roles/verify/inventory/tasks/main.yml b/roles/verify/inventory/tasks/main.yml index 26af2c35..cb579c60 100644 --- a/roles/verify/inventory/tasks/main.yml +++ b/roles/verify/inventory/tasks/main.yml @@ -44,6 +44,7 @@ assert: that: >- {{ groups.tls | difference(cluster_hosts) | length == 0 }} + fail_msg: "The following hosts are different between tls hosts and cluster hosts: {{ groups.tls | difference(cluster_hosts) }}" when: - krb5_kdc_type == "Red Hat IPA" - not (skip_ipa_signing | default(false)) diff --git a/tests/config.yml b/tests/config.yml new file mode 100644 index 00000000..db18769d --- /dev/null +++ b/tests/config.yml @@ -0,0 +1,21 @@ +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Support for this feature was first added in ansible-core 2.12. +# See: +# - https://github.com/ansible-collections/overview/issues/45#issuecomment-827853900 +# - https://github.com/ansible/ansible/blob/devel/test/lib/ansible_test/config/config.yml + +modules: + python_requires: '>=3.6' diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 00000000..96e9c67d --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys +import pytest + +@pytest.fixture(autouse=True) +def skip_python(): + if sys.version_info < (3, 6): + pytest.skip('Skipping on Python %s. cloudera.cloud supports Python 3.6 and higher.' % sys.version) diff --git a/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py b/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py new file mode 100644 index 00000000..5ee00fe6 --- /dev/null +++ b/tests/unit/plugins/modules/cm_endpoint_info/test_cm_endpoint_info_i.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pytest +import re +import unittest + +from ansible_collections.cloudera.cluster.plugins.modules import cm_endpoint_info +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args + + +@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +class TestCMVersionIntegration(ModuleTestCase): + + def test_host_discovery(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "port": "7180", + "verify_tls": "no", + "debug": "yes" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_endpoint_info.main() + + self.assertEquals(e.value.args[0]['endpoint'], "https://" + os.getenv('CM_HOST') + ":" + os.getenv('CM_PORT_TLS') + "/api/" + os.getenv('CM_VERSION')) + + def test_direct_endpoint(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "url": "http://not.supported", + "verify_tls": "no", + "debug": "yes" + }) + + with pytest.raises(AnsibleFailJson) as e: + cm_endpoint_info.main() + + self.assertRegexpMatches(e.value.args[0]['msg'], "^Unsupported parameters") + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py b/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py new file mode 100644 index 00000000..c98a8594 --- /dev/null +++ b/tests/unit/plugins/modules/cm_resource/test_cm_resource_i.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pytest +import unittest + +from ansible_collections.cloudera.cluster.plugins.modules import cm_resource +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args + + +@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +class TestCMResourceIntegration(ModuleTestCase): + + def test_post(self): + create_module_args = { + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "method": "POST", + "path": "/users", + "body": { + "items": [ + { + "name": "unit_test", + "password": "UnsecurePassword" + } + ] + } + } + + update_module_args = { + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "method": "PUT", + "path": "/users/unit_test", + "body": { + "authRoles": [{ "name": "ROLE_LIMITED" }] + } + } + + delete_module_args = { + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "method": "DELETE", + "path": "/users/unit_test" + } + + # Create + setup_module_args(create_module_args) + with pytest.raises(AnsibleExitJson) as e: + cm_resource.main() + self.assertIsInstance(e.value.args[0]['resources'], list) + + # Create fail on duplicate + setup_module_args(create_module_args) + with pytest.raises(AnsibleFailJson) as e: + cm_resource.main() + self.assertEquals(e.value.args[0]['status_code'], 400) + + # Update + setup_module_args(update_module_args) + with pytest.raises(AnsibleExitJson) as e: + cm_resource.main() + self.assertIsInstance(e.value.args[0]['resources'], list) + + # Delete + setup_module_args(delete_module_args) + with pytest.raises(AnsibleExitJson) as e: + cm_resource.main() + self.assertIsInstance(e.value.args[0]['resources'], list) + + # Delete fail on existence + setup_module_args(delete_module_args) + with pytest.raises(AnsibleFailJson) as e: + cm_resource.main() + self.assertEquals(e.value.args[0]['status_code'], 404) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py b/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py new file mode 100644 index 00000000..2d644e48 --- /dev/null +++ b/tests/unit/plugins/modules/cm_resource_info/test_cm_resource_info_i.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pytest +import re +import unittest + +from ansible_collections.cloudera.cluster.plugins.modules import cm_resource_info +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args + + +@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +class TestCMResourceInfoIntegration(ModuleTestCase): + + def test_list(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "path": "/clusters" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_resource_info.main() + + self.assertIsInstance(e.value.args[0]['resources'], list) + + def test_item(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "path": "/cm/license" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_resource_info.main() + + self.assertIsInstance(e.value.args[0]['resources'], list) + + def test_invalid_host(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": "nope", + "verify_tls": "no", + "debug": "yes", + "path": "/cm/license" + }) + + with pytest.raises(AnsibleFailJson) as e: + cm_resource_info.main() + + self.assertRegexpMatches(e.value.args[0]['msg'], "nodename nor servname provided, or not known") + + def test_invalid_path(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "path": "/cm/licenseZ" + }) + + with pytest.raises(AnsibleFailJson) as e: + cm_resource_info.main() + + self.assertRegexpMatches(e.value.args[0]['msg'], "^API error: Not Found$") + + def test_query_params(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "verify_tls": "no", + "debug": "yes", + "path": "/tools/echo", + "query": { + "message": "foobarbaz" + }, + "field": "message" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_resource_info.main() + + self.assertRegexpMatches(e.value.args[0]['resources'][0], "^foobarbaz$") + + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py b/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py new file mode 100644 index 00000000..8654e000 --- /dev/null +++ b/tests/unit/plugins/modules/cm_version_info/test_cm_version_info_i.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pprint +import pytest +import unittest + +from ansible_collections.cloudera.cluster.plugins.modules import cm_version_info +from ansible_collections.cloudera.cluster.tests.unit.plugins.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, setup_module_args + + +@unittest.skipUnless(os.getenv('CM_USERNAME'), "Cloudera Manager access parameters not set") +class TestCMVersionIntegration(ModuleTestCase): + + def test_host_discovery(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "host": os.getenv('CM_HOST'), + "port": "7180", + "verify_tls": "no", + "debug": "yes" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_version_info.main() + + self.assertEquals(e.value.args[0]['cm']['version'], "7.6.5") + + def test_direct_endpoint(self): + setup_module_args({ + "username": os.getenv('CM_USERNAME'), + "password": os.getenv('CM_PASSWORD'), + "url": "https://" + os.getenv('CM_HOST') + ":" + os.getenv('CM_PORT_TLS') + "/api/" + os.getenv('CM_VERSION'), + "verify_tls": "no", + "debug": "yes" + }) + + with pytest.raises(AnsibleExitJson) as e: + cm_version_info.main() + + self.assertEquals(e.value.args[0]['cm']['version'], "7.6.5") + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/unit/plugins/modules/utils.py b/tests/unit/plugins/modules/utils.py new file mode 100644 index 00000000..3624d05b --- /dev/null +++ b/tests/unit/plugins/modules/utils.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import json +import unittest + +from unittest.mock import patch +from ansible.module_utils import basic +from ansible.module_utils.common.text.converters import to_bytes + +def setup_module_args(args): + args = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + basic._ANSIBLE_ARGS = to_bytes(args) + + +class AnsibleExitJson(Exception): + pass + + +class AnsibleFailJson(Exception): + pass + + +def exit_json(*args, **kwargs): + if 'changed' not in kwargs: + kwargs['changed'] = False + raise AnsibleExitJson(kwargs) + +def fail_json(*args, **kwargs): + kwargs['failed'] = True + raise AnsibleFailJson(kwargs) + + +class ModuleTestCase(unittest.TestCase): + def setUp(self): + self.mock_module = patch.multiple(basic.AnsibleModule, + exit_json=exit_json, + fail_json=fail_json) + self.mock_module.start() + self.mock_sleep = patch('time.sleep') + self.mock_sleep.start() + setup_module_args({}) + self.addCleanup(self.mock_module.stop) + self.addCleanup(self.mock_sleep.stop) diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt new file mode 100644 index 00000000..63aa668a --- /dev/null +++ b/tests/unit/requirements.txt @@ -0,0 +1,15 @@ +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cm_client \ No newline at end of file