From dde33a9cc7922d8893e77f517685eb8dce4714b7 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 8 Mar 2023 16:33:08 -0500 Subject: [PATCH 1/4] Update PvC prerequisites and Control Plane (#89) * Initial version for CDP Public Cloud setup and teardown using definition-based infrastructure * Migrate roles from cloudera-deploy to cloudera.exe collection, as relative imports are difficult in Ansible, but FQCN imports are always good * Migrate WIP example definitions to separate repo * Remove most uses of push-down Ansible tags as they confuse even the most experienced Ansible user * Rename several playbooks to be more explanatory of what service they work on * Split playbooks into separate deployment areas to allow users to run adhoc or in sequence via main.yml * Update private cloud cluster deployment to include necessary steps for ECS deployment * Refactor main to simplify some pathways for better readability * Rename iptables role to be generic prereqs for pvc_ecs * Gather facts necessary for cluster deployment * Update pvc_cp_setup.yml so it runs independent of main.yml * Break pvc playbooks out into groups according to functional area to allow independent execution * Remove separate pvc-ds playbook as tags and switches better control functionality structured within the main deployment process * Add teardown play for ECS clusters Signed-off-by: Daniel Chaffelson Co-authored-by: Webster Mudge Signed-off-by: Webster Mudge --- LICENSE | 2 +- cloud.yml | 98 --- cluster.yml | 603 ------------------ examples/sandbox/definition.yml | 2 +- main.yml | 128 ++-- pbc_setup.yml | 73 +++ pbc_teardown.yml | 50 ++ pvc_base_postfix.yml | 197 ++++++ pvc_base_prereqs_ext.yml | 302 +++++++++ pvc_base_prereqs_int.yml | 247 +++++++ pvc_base_setup.yml | 67 ++ pvc_base_teardown.yml | 98 +++ .../defaults/basic_cluster.yml | 68 -- roles/cloudera_deploy/defaults/main.yml | 55 -- .../tasks/auto_terraform_state.yml | 48 -- .../tasks/clean_dynamic_inventory.yml | 31 - .../tasks/distribute_facts_to_inventory.yml | 69 -- roles/cloudera_deploy/tasks/init.yml | 483 -------------- .../tasks/inject_download_mirror.yml | 129 ---- .../tasks/persist_dynamic_inventory.yml | 62 -- .../tasks/populate_download_mirror.yml | 136 ---- .../tasks/prepare_download_mirror.yml | 118 ---- .../tasks/refresh_inventory.yml | 51 -- 23 files changed, 1108 insertions(+), 2009 deletions(-) delete mode 100644 cloud.yml delete mode 100644 cluster.yml create mode 100644 pbc_setup.yml create mode 100644 pbc_teardown.yml create mode 100644 pvc_base_postfix.yml create mode 100644 pvc_base_prereqs_ext.yml create mode 100644 pvc_base_prereqs_int.yml create mode 100644 pvc_base_setup.yml create mode 100644 pvc_base_teardown.yml delete mode 100644 roles/cloudera_deploy/defaults/basic_cluster.yml delete mode 100644 roles/cloudera_deploy/defaults/main.yml delete mode 100644 roles/cloudera_deploy/tasks/auto_terraform_state.yml delete mode 100644 roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml delete mode 100644 roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml delete mode 100644 roles/cloudera_deploy/tasks/init.yml delete mode 100644 roles/cloudera_deploy/tasks/inject_download_mirror.yml delete mode 100644 roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml delete mode 100644 roles/cloudera_deploy/tasks/populate_download_mirror.yml delete mode 100644 roles/cloudera_deploy/tasks/prepare_download_mirror.yml delete mode 100644 roles/cloudera_deploy/tasks/refresh_inventory.yml diff --git a/LICENSE b/LICENSE index 0324e14..7770734 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2021 Cloudera, Inc. +Copyright 2022 Cloudera, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cloud.yml b/cloud.yml deleted file mode 100644 index 43d48cd..0000000 --- a/cloud.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This play is imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml -- name: Marshal Cloud Deployment - hosts: localhost - environment: "{{ globals.env_vars }}" - gather_facts: yes - tasks: - - name: Import the Sequence execution Role for Cloud Run - ansible.builtin.import_role: - name: cloudera.exe.sequence - when: init__call_cloud_role | bool - - - name: Persist Dynamic Inventory to Definition Path if exists - tags: always - when: - - infra__dynamic_inventory_host_entries is defined - - infra__dynamic_inventory_host_entries | length > 0 - - init__dynamic_inventory_template is defined - - init__dynamic_inventory_template | length > 0 - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: persist_dynamic_inventory - apply: - tags: always - -# TODO: Set timeout waiting for utility VM to be ready -- name: Process Download Mirror on Utility VM - hosts: cldr_utility - tags: always - gather_facts: yes - tasks: - - name: Fetch necessary variables from Ansible Controller - ansible.builtin.set_fact: - globals: "{{ hostvars['localhost']['globals'] }}" - init__download_mirror_artefact: "{{ hostvars['localhost']['init__download_mirror_artefact'] }}" - - - name: Prepare Cloudera Subscription Credentials - ansible.builtin.include_role: - name: cloudera.cluster.deployment.credential - when: globals.cloudera_license_file is defined - vars: - cloudera_manager_license_file: "{{ globals.cloudera_license_file }}" - - - name: Populate the Download Mirror with new files - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: populate_download_mirror - when: globals.download_mirror_targets | length > 0 - -- name: Update relevant Download Mirror Cache listing - hosts: localhost - tags: always - gather_facts: no - tasks: - - name: Refresh Listing of target cache contents - when: - - init__download_mirror_bucket_name is defined - - "'teardown' not in ansible_run_tags" - register: __infra_download_mirror_listing - failed_when: - - __download_mirror_lookup_initial.s3_keys is not defined - - "'cannot be found' not in __download_mirror_lookup_initial.msg" - amazon.aws.aws_s3: - bucket: "{{ init__download_mirror_bucket_name }}" - mode: list - - - name: Prepare updated Download Mirror contents as URLs - when: __infra_download_mirror_listing.s3_keys is defined - loop: "{{ __infra_download_mirror_listing.s3_keys }}" - loop_control: - loop_var: __download_mirror_s3_urls_item - ansible.builtin.set_fact: - __download_mirror_url_listing: "{{ __download_mirror_url_listing | default([]) + [['https:/', init__download_mirror_bucket_name + '.s3.amazonaws.com', __download_mirror_s3_urls_item ] | join('/') ] }}" - - - name: Persist Download Mirror to Definition path - when: - - __download_mirror_url_listing is defined - - __download_mirror_url_listing | length > 0 - community.general.ini_file: - path: "{{ init__download_mirror_artefact }}" - section: "{{ globals.infra_type }}:{{ globals.region }}" - option: "{{ init__download_mirror_bucket_name }}" - value: "{{ __download_mirror_url_listing }}" \ No newline at end of file diff --git a/cluster.yml b/cluster.yml deleted file mode 100644 index 3cc26a2..0000000 --- a/cluster.yml +++ /dev/null @@ -1,603 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Playbook is specifically for Deploying Cloudera Clusters -# Edit with extreme caution -# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml - -# STARTBLOCK # Init run -- name: Init Cluster Deployment Tasks - hosts: localhost - tags: always - gather_facts: no - tasks: - # TODO: Move to dependent roles with delegation if necessary - - name: Create local temp directories - file: "{{ __dir }}" - loop: - - path: "{{ local_temp_dir }}/csrs" - state: directory - - path: "{{ local_temp_dir }}/certs" - state: directory - loop_control: - loop_var: __dir - -- name: Check Inventory Connectivity - hosts: all - tags: always - gather_facts: no - tasks: - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - - - name: Gather facts from connected inventory - setup: - -# ENDBLOCK # Init run -# STARTBLOCK # Verify Inventory and Definition -# Moved from tasks/group_hosts and tasks/group_hosts_by_tls -- name: Group hosts using host template information - hosts: all - gather_facts: no - roles: - - cloudera.cluster.deployment.groupby - tags: - - always - -- name: Verify inventory [verify_inventory] - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.verify.inventory - tags: - - verify - - verify_inventory - - default_cluster - - full_cluster - -- name: Verify definition [verify_definition] - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.verify.definition - tags: - - verify - - verify_definition - - default_cluster - - full_cluster - -# Moved before parcel verification to allow rehosting -- name: Install custom parcel repository - hosts: custom_repo - become: yes - roles: - - cloudera.cluster.infrastructure.custom_repo - tags: - - custom_repo - - default_cluster - - full_cluster - -# Moved from verify_parcels to reduce duplication -- name: Verify definition [verify_parcels_and_roles] - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.verify.parcels_and_roles - tags: - - verify - - verify_parcels - - default_cluster - - full_cluster - -# ENDBLOCK # Verify Inventory and Definition -# STARTBLOCK # Prepare Nodes - -- name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster, ca_server - become: yes - roles: - - cloudera.cluster.prereqs.os - tags: - - os - - default_cluster - - full_cluster - -- name: Create local user accounts - hosts: cloudera_manager, cluster, tls - become: yes - gather_facts: no - roles: - - cloudera.cluster.prereqs.user_accounts - tags: - - users - - default_cluster - - full_cluster - -- name: Install JDK - hosts: cloudera_manager, cluster, tls - become: yes - roles: - - cloudera.cluster.prereqs.jdk - tags: - - jdk - - default_cluster - - full_cluster - -- name: Install MySQL Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.mysql_connector - when: database_type == 'mysql' or database_type == 'mariadb' - tags: - - mysql_connector - - full_cluster - -- name: Install Oracle Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.oracle_connector - when: database_type == 'oracle' - tags: - - oracle_connector - - full_cluster - -- name: Install PostgreSQL Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.postgresql_connector - when: database_type == 'postgresql' - tags: - - postgresql_connector - - full_cluster - -# ENDBLOCK # Prepare Nodes -# STARTBLOCK # Create Cluster Service Infrastructure - -- name: Install KDC - hosts: krb5_server - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_server - tags: - - security - - kerberos - - tls - - full_cluster - -- name: Setup KRB5 clients - hosts: cloudera_manager, cluster - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_client - when: "'krb5_server' in groups" - tags: - - security - - kerberos - - tls - - full_cluster - -- name: Install CA server - hosts: ca_server - become: yes - roles: - - cloudera.cluster.infrastructure.ca_server - tags: - - security - - tls - - full_cluster - -- name: Install HAProxy - hosts: haproxy - become: yes - roles: - - cloudera.cluster.infrastructure.haproxy - tags: - - ha - - full_cluster - -# ENDBLOCK # Create Cluster Service Infrastructure -# STARTBLOCK # Prepare TLS - -- name: Build TLS keystores and truststores - hosts: tls - become: yes - gather_facts: no - roles: - - cloudera.cluster.security.tls_generate_csr - - cloudera.cluster.security.tls_signing - - cloudera.cluster.security.tls_install_certs - vars: - local_certs_dir: "{{ local_temp_dir }}/certs" - local_csrs_dir: "{{ local_temp_dir }}/csrs" - tags: - - tls - - full_cluster - -- name: Delete temp directories - hosts: localhost - become: no - gather_facts: no - tasks: - - file: - path: "{{ [local_temp_dir, dir] | path_join }}" - state: absent - loop: - - csrs - - certs - loop_control: - loop_var: dir - tags: - - always - -# ENDBLOCK # Prepare TLS -# STARTBLOCK # Install Cluster Service Infrastructure II -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - cloudera.cluster.infrastructure.rdbms - tags: - - database - - default_cluster - - full_cluster -# ENDBLOCK # Install Cluster Service Infrastructure II -# STARTBLOCK # NiFi TLS - -- name: Setup symlinks for NiFi TLS keystore and truststore - hosts: cluster - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.security.tls_nifi - when: > - (tls | default(False) - or manual_tls_cert_distribution | default(False)) - and not (autotls | default(False)) - -# ENDBLOCK # NiFi TLS -# STARTBLOCK # Install Cloudera Manager - -- name: Install Cloudera Manager daemons - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.daemons - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager server - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.server - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager License - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.license - tags: - - cm - - license - - default_cluster - - full_cluster - -- name: Install Cloudera Manager agents - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.agent - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server for TLS - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.server_tls - when: tls | default(False) or manual_tls_cert_distribution | default(False) - tags: - - tls - - cm - - full_cluster - -- name: Configure Cloudera Manager agents - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.cloudera_manager.agent_config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager hosts config - hosts: localhost - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.hosts_config - tags: - - cm - - default_cluster - - full_cluster - -- name: Restart agents if the parcel directory has been changed - hosts: cloudera_manager, cluster - gather_facts: no - roles: - - role: cloudera.cluster.operations.restart_agents - when: - - definition.hosts.configs is defined - - "'parcels_directory' in definition.hosts.configs" - - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat - when: - - definition.hosts.configs is defined - - "'parcels_directory' in definition.hosts.configs" - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager auth and accounts - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.external_auth - - role: cloudera.cluster.cloudera_manager.external_account - tags: - - cm - - full_cluster - -- name: Configure Cloudera Manager Password - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.admin_password.set - tags: - - cm - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cloudera Manager -# STARTBLOCK # Cloudera Manager Password - -- name: Check Cloudera Manager admin password - hosts: cloudera_manager, cluster - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.api_client - tags: - - cm - - default_cluster - - full_cluster - -# ENDBLOCK # Cloudera Manager Password -# STARTBLOCK # Prepare Security -# Insert AutoTLS with tag -- name: Enable Auto-TLS - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.autotls - tags: - - autotls - - never - -- name: Install pre-requisite packages for Kerberos - hosts: cloudera_manager, cluster - become: yes - roles: - - role: cloudera.cluster.prereqs.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - prereqs - - full_cluster - -- name: Configure Cloudera Manager server for Kerberos - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - full_cluster - -# ENDBLOCK # Prepare Security -# STARTBLOCK # Install Cluster - -- name: Restart Cloudera Manager Agents - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.operations.restart_agents - tags: - - never - - restart_agents - -- name: Ensure that the agents are heartbeating - hosts: cloudera_manager, cluster - gather_facts: yes - any_errors_fatal: yes - roles: - - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat - when: cloudera_manager_agent_wait_for_heartbeat | default(True) - tags: - - heartbeat - - default_cluster - - full_cluster - -- name: Deploy Cloudera Management Service - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.deployment.services.mgmt - tags: - - mgmt - - default_cluster - - full_cluster - -- name: Preload parcels from custom repo to Cloudera Manager - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.preload_parcels - when: "'custom_repo' in groups" - tags: - - preload_parcels - - default_cluster - - full_cluster - -- name: Deploy clusters - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.deployment.cluster - tags: - - cluster - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cluster -# STARTBLOCK # Setup HDFS Encryption - -- name: Setup KTS HA - hosts: localhost - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.deployment.services.kts_high_availability - when: - - "'kts_active' in groups" - - "'kts_passive' in groups" - tags: - - kts - - full_cluster - -- name: Handle KMS services - hosts: localhost - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.deployment.services.kms - when: "'kms_servers' in groups" - - role: cloudera.cluster.deployment.services.kms_ha - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Handle KMS services - hosts: cloudera_manager - gather_facts: no - become: no - roles: - - role: cloudera.cluster.operations.refresh_ranger_kms_repo - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Restart and re-deploy stale client configs - hosts: localhost - gather_facts: no - roles: - - role: cloudera.cluster.operations.restart_stale - when: "'kms_servers' in groups" - vars: - client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" - tags: - - kms - - restart_stale - - full_cluster - -# ENDBLOCK # Setup HDFS Encryption -# STARTBLOCK # Teardown -# Teardown CA - -- name: Teardown CA server - hosts: ca_server - gather_facts: no - become: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.cluster.infrastructure.ca_certs - tasks_from: clean.yml - tags: - - teardown_ca - - teardown_all - - never - -- name: Teardown security artifact directories - hosts: tls - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.security.tls_clean - when: "'tls' in groups" - tags: - - teardown_tls - - teardown_all - - never - -# Teardown Cluster - -- name: Teardown - hosts: all - gather_facts: no - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.teardown - tags: - - teardown_cluster - - teardown_all - - never - -# ENDBLOCK # Teardown -# End run -### diff --git a/examples/sandbox/definition.yml b/examples/sandbox/definition.yml index 797774c..ebc392e 100644 --- a/examples/sandbox/definition.yml +++ b/examples/sandbox/definition.yml @@ -19,5 +19,5 @@ datahub: - include: "datahub_streams_messaging_light.j2" use_default_cluster_definition: yes -use_download_mirror: yes +use_auto_repo_mirror: yes preload_cm_parcel_repo: yes \ No newline at end of file diff --git a/main.yml b/main.yml index 323aa79..979499d 100644 --- a/main.yml +++ b/main.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2022 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,72 +14,88 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Primary entrypoint for all CDP Installations +# Primary entrypoint for all CDP installations (Public Cloud, Private Cloud) +# +# There are five "core" runlevels: +# - validate +# - initialize +# - infra +# - plat +# - run +# - teardown +# +# In this context, 'run' means 'runtime' or 'application'. The platform level +# is now both CDP Public Cloud SDX and Data Services, which is a change +# from before where 'plat' referred to the DL and SDX and 'run' referred to the +# Data Services. +# +# Individual Data Services (and Datahubs) should be tagged _within_ the +# 'cloudera.exe.*' roles themselves: +# - de +# - df +# - dh +# - dw +# - ml +# - opdb -- name: Init Cloudera-Deploy Run +- name: Initialize Cloudera Deploy Run hosts: localhost - tags: always + connection: local gather_facts: yes + tags: always tasks: - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: init + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined -- name: Import the Cloud Interactions Playbook - import_playbook: cloud.yml +- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_teardown.yml'] | path_join }}" + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_teardown -- name: Teardown Cleanup - hosts: localhost - tags: [teardown,never] - gather_facts: yes - tasks: - - name: Remove current Dynamic Inventory file from Definition Path if exists - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: clean_dynamic_inventory +- ansible.builtin.import_playbook: pbc_teardown.yml + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + - hostvars.localhost.init__call_cdp_pbc == True - - name: Remove Terraform remote state resources if requested - when: - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: auto_terraform_state +- ansible.builtin.import_playbook: pvc_base_teardown.yml + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + - hostvars.localhost.init__call_cdp_pvc == True -- name: Prepare for Cloudera Cluster Run - hosts: localhost - tags: always - gather_facts: yes - tasks: - - name: Load Static Inventory file if present - when: mgmt is defined | bool - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: refresh_inventory - vars: - include_inventory_file: "{{ init__dynamic_inventory_artefact }}" +- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_setup.yml'] | path_join }}" + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_runtime + +- ansible.builtin.import_playbook: pbc_setup.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pbc == True + +- ansible.builtin.import_playbook: pvc_base_prereqs_ext.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True + +- ansible.builtin.import_playbook: pvc_base_prereqs_int.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True - - name: Inject Download Mirror if requested - when: - - "'teardown' not in ansible_run_tags" - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: inject_download_mirror +- ansible.builtin.import_playbook: pvc_base_setup.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True - - name: Distribute Facts to Inventory Hosts - when: - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: distribute_facts_to_inventory +- ansible.builtin.import_playbook: pvc_base_postfix.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True -- name: Import the Cluster Interactions Playbook +- ansible.builtin.import_playbook: "{{ [definition_path, 'post_setup.yml'] | path_join }}" when: - - "'cluster' in groups" - - mgmt is defined | bool - import_playbook: cluster.yml + - hostvars.localhost.run_runtime -- name: Execute the Application Playbook - import_playbook: "{{ [definition_path, 'application.yml'] | path_join }}" +- ansible.builtin.import_playbook: "{{ [definition_path, 'post_teardown.yml'] | path_join }}" + when: + - hostvars.localhost.run_teardown diff --git a/pbc_setup.yml b/pbc_setup.yml new file mode 100644 index 0000000..39eab83 --- /dev/null +++ b/pbc_setup.yml @@ -0,0 +1,73 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up CDP Public Cloud + hosts: localhost + connection: local + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# Inherits 'plat' and 'run' from playbook import +# Applies 'validate', and 'initialize' via the role imports +# By using imports vs includes for roles, we can propagate +# tags via --list-tags + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Validate Platform configuration + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: validate + tags: + - validate + + - name: Validate Data Services configuration + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: validate + tags: + - validate + + - name: Initialize Platform setup + #when: sequence__setup_plat | bool + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_setup + tags: + - initialize + + - name: Set up Platform + #when: sequence__setup_plat | bool + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: setup + + - name: Initialize Data Services setup + #when: sequence__setup_runtime | bool + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_setup + tags: + - initialize + + - name: Set up Data Services + #when: sequence__setup_runtime | bool + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: setup diff --git a/pbc_teardown.yml b/pbc_teardown.yml new file mode 100644 index 0000000..2f63635 --- /dev/null +++ b/pbc_teardown.yml @@ -0,0 +1,50 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Tear down CDP Public Cloud + hosts: localhost + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# Inherits 'infra', 'teardown', and 'never' from the playbook import +# Does not apply other tags via the role imports +# By using imports vs includes for roles, we can propagate +# tags via --list-tags + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Initialize Data Services teardown + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_teardown + + - name: Tear down Data Services + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: teardown + + - name: Initialize Platform teardown + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_teardown + + - name: Tear down Platform + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: teardown diff --git a/pvc_base_postfix.yml b/pvc_base_postfix.yml new file mode 100644 index 0000000..a1f4e3e --- /dev/null +++ b/pvc_base_postfix.yml @@ -0,0 +1,197 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution +# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run +# STARTBLOCK # Fix Auto-TLS + +- name: Auto-TLS Services Setup + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.cms_tls + when: autotls is defined and autotls == True + tags: + - autotls + - never + +# ENDBLOCK # Fix Auto-TLS +# STARTBLOCK # Setup HDFS Encryption + +- name: Setup KTS HA + hosts: localhost + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.deployment.services.kts_high_availability + when: + - "'kts_active' in groups" + - "'kts_passive' in groups" + tags: + - kts + - full_cluster + +- name: Handle KMS services + hosts: localhost + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.deployment.services.kms + when: "'kms_servers' in groups" + - role: cloudera.cluster.deployment.services.kms_ha + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Handle KMS services + hosts: cloudera_manager + gather_facts: no + become: no + roles: + - role: cloudera.cluster.operations.refresh_ranger_kms_repo + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Restart and re-deploy stale client configs + hosts: localhost + gather_facts: no + roles: + - role: cloudera.cluster.operations.restart_stale + when: "'kms_servers' in groups" + vars: + client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" + tags: + - kms + - restart_stale + - full_cluster + + # ENDBLOCK # Setup HDFS Encryption + # STARTBLOCK # WXM Setup + +- name: Handle WXM Setup + hosts: "{{ tp_host | default('cluster_master_nodes[0]') }}" + gather_facts: yes + tags: + - wxm + - full_cluster + tasks: + - name: Setup WXM + when: + - use_wxm | default(False) + - altus_key_id | length > 0 + - altus_private_key | length > 0 + import_role: + name: cloudera.cluster.deployment.services.wxm + +# ENDBLOCK # WXM Setup + +- name: Post-Install for PvC on all cluster hosts + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: yes + tags: + - pvc + tasks: + - name: Add missing ExtJS for Oozie UI + include_role: + name: cloudera.cluster.config.services.oozie_ui + when: oozie_service_exists | default(false) + +- name: Post-Install Cloudera Manager and Cluster + hosts: cloudera_manager + gather_facts: yes + tags: + - pvc + tasks: + - name: Refresh CM Services Info + include_role: + name: cloudera.cluster.cloudera_manager.services_info + public: yes + vars: + cluster_name: "{{ cluster_name_base }}" + + - name: Fix Hue ticket lifetime for Free IPA + include_role: + name: cloudera.cluster.config.services.hue_ticket_lifetime + when: + - hue_service_exists | default(false) + - krb5_kdc_type == 'Red Hat IPA' + + - name: Set Cloudera Manager session timeout to 30d + include_role: + name: cloudera.cluster.cloudera_manager.session_timeout + + - name: Create missing Solr plugin for Ranger + include_role: + name: cloudera.cluster.config.services.solr_ranger_plugin + when: + - ranger_service_exists | default(false) + - solr_service_exists | default(false) + + - name: Add Solr urls to Knox + include_role: + name: cloudera.cluster.config.services.solr_knox + when: knox_service_exists | default(false) and solr_service_exists | default(false) + + - name: Add missing TLS values for KMS + when: + - kms_service_exists | default(False) + - (autotls | default(False)) + include_role: + name: cloudera.cluster.config.services.kms_tls + + - name: Ranger extra policies to have enough rights with basic accounts + when: ranger_service_exists | default(false) + include_role: + name: cloudera.cluster.config.services.ranger_pvc_default_policies + +# End run +### \ No newline at end of file diff --git a/pvc_base_prereqs_ext.yml b/pvc_base_prereqs_ext.yml new file mode 100644 index 0000000..b41284a --- /dev/null +++ b/pvc_base_prereqs_ext.yml @@ -0,0 +1,302 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + +# STARTBLOCK # Verify Inventory and Definition +- name: Verify inventory [verify_inventory] + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.verify.inventory + tags: + - verify + - verify_inventory + - default_cluster + - full_cluster + +- name: Verify definition [verify_definition] + hosts: cloudera_manager + gather_facts: no + roles: + - cloudera.cluster.verify.definition + tags: + - verify + - verify_definition + - default_cluster + - full_cluster + +# Moved before parcel verification to allow rehosting +- name: Install custom parcel repository + hosts: custom_repo + become: yes + roles: + - cloudera.cluster.infrastructure.custom_repo + tags: + - custom_repo + - default_cluster + - full_cluster + +# Moved from verify_parcels to reduce duplication +- name: Verify definition [verify_parcels_and_roles] + hosts: cloudera_manager + gather_facts: no + roles: + - cloudera.cluster.verify.parcels_and_roles + tags: + - verify + - verify_parcels + - default_cluster + - full_cluster + +# ENDBLOCK # Verify Inventory and Definition +# STARTBLOCK # Prepare Nodes + +- name: Apply OS pre-requisite configurations + hosts: cloudera_manager, cluster, ca_server, ecs_nodes + become: yes + roles: + - cloudera.cluster.prereqs.os + tags: + - os + - default_cluster + - full_cluster + +- name: Apply OS Prereqs to ECS Nodes + hosts: ecs_nodes + gather_facts: yes + become: yes + tags: + - pvc + - os + - default_cluster + - full_cluster + tasks: + - name: Setup OS Prereqs for ECS Nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.pvc_ecs + + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + +- name: Create local user accounts + hosts: cloudera_manager, cluster + become: yes + gather_facts: no + roles: + - cloudera.cluster.prereqs.user_accounts + tags: + - users + - default_cluster + - full_cluster + +- name: Create local users on ECS Nodes + hosts: ecs_nodes + gather_facts: yes + tags: + - pvc + - users + - default_cluster + - full_cluster + tasks: + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + +# TODO: add pvc tag to all ecs_nodes plays + +- name: Install JDK + hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes + become: yes + roles: + - cloudera.cluster.prereqs.jdk + tags: + - jdk + - security + - free_ipa + - kerberos + - tls + - default_cluster + - full_cluster + +# DB Connectors +- name: Install MySQL Connector + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.mysql_connector + when: database_type == 'mysql' or database_type == 'mariadb' + tags: + - mysql_connector + - full_cluster + +- name: Install Oracle Connector + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.oracle_connector + when: database_type == 'oracle' + tags: + - oracle_connector + - full_cluster + +# ENDBLOCK # Prepare Nodes +# STARTBLOCK # Install Cluster Service Infrastructure II + +- name: Install RDBMS + hosts: db_server + become: yes + roles: + - cloudera.cluster.infrastructure.rdbms + tags: + - database + - default_cluster + - full_cluster + +# ENDBLOCK # Install Cluster Service Infrastructure II +# STARTBLOCK # Create Cluster Service Infrastructure + +- name: Install Kerberos Server + hosts: krb5_server + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_server + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Setup KRB5 clients + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_client + when: "'krb5_server' in groups" + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Install CA server + hosts: ca_server + become: yes + roles: + - cloudera.cluster.infrastructure.ca_server + tags: + - security + - tls + - full_cluster + +- name: Install HAProxy + hosts: haproxy + become: yes + roles: + - cloudera.cluster.infrastructure.haproxy + tags: + - ha + - full_cluster + +# ENDBLOCK # Create Cluster Service Infrastructure +# STARTBLOCK # Prepare TLS + +- name: Build TLS keystores and truststores + hosts: tls + become: yes + gather_facts: no + roles: + - cloudera.cluster.security.tls_generate_csr + - cloudera.cluster.security.tls_signing + - cloudera.cluster.security.tls_install_certs + vars: + local_certs_dir: "{{ local_temp_dir }}/certs" + local_csrs_dir: "{{ local_temp_dir }}/csrs" + tags: + - tls + - full_cluster + +- name: Delete temp directories + hosts: localhost + become: no + gather_facts: no + tasks: + - file: + path: "{{ [local_temp_dir, dir] | path_join }}" + state: absent + loop: + - csrs + - certs + loop_control: + loop_var: dir + tags: + - always + +# ENDBLOCK # Prepare TLS +# STARTBLOCK # NiFi TLS + +- name: Setup symlinks for NiFi TLS keystore and truststore + hosts: cluster + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.security.tls_nifi + when: > + (tls | default(False) + or manual_tls_cert_distribution | default(False)) + and not (autotls | default(False)) + +# ENDBLOCK # NiFi TLS diff --git a/pvc_base_prereqs_int.yml b/pvc_base_prereqs_int.yml new file mode 100644 index 0000000..01833bc --- /dev/null +++ b/pvc_base_prereqs_int.yml @@ -0,0 +1,247 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + +# STARTBLOCK # Install Cloudera Manager + +- name: Install Cloudera Manager daemons + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.daemons + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager server + hosts: cloudera_manager + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.server + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager License + hosts: cloudera_manager + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.license + tags: + - cm + - license + - default_cluster + - full_cluster + +- name: Install Cloudera Manager agents + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.agent + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server for TLS + hosts: cloudera_manager + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.server_tls + when: tls | default(False) or manual_tls_cert_distribution | default(False) + tags: + - tls + - cm + - full_cluster + +- name: Configure Cloudera Manager agents + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.cloudera_manager.agent_config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager auth and accounts + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.external_auth + - role: cloudera.cluster.cloudera_manager.external_account + tags: + - cm + - full_cluster + +# ENDBLOCK # Install Cloudera Manager +# STARTBLOCK # Cloudera Manager Password + +- name: Configure Cloudera Manager Password + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.admin_password.set + tags: + - cm + - default_cluster + - full_cluster + +- name: Check Cloudera Manager admin password + hosts: cloudera_manager, cluster + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.api_client + tags: + - cm + - default_cluster + - full_cluster + +# ENDBLOCK # Cloudera Manager Password +# STARTBLOCK # Prepare Security + +- name: Enable Auto-TLS + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.autotls + when: autotls is defined and autotls == True + tags: + - autotls + - full_cluster + +- name: Install pre-requisite packages for Kerberos + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + roles: + - role: cloudera.cluster.prereqs.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - prereqs + - full_cluster + +- name: Configure Cloudera Manager server for Kerberos + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - full_cluster + +# ENDBLOCK # Prepare Security +# STARTBLOCK # Configure CM + +- name: Restart Cloudera Manager Agents + hosts: cloudera_manager, cluster + gather_facts: no + become: yes + tasks: + - name: Restart Cloudera Manager Agents + meta: noop + notify: + - restart cloudera-scm-agent + tags: + - never + - restart_agents + +- name: Ensure that the agents are heartbeating + hosts: cloudera_manager, cluster + gather_facts: yes + any_errors_fatal: yes + roles: + - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat + when: cloudera_manager_agent_wait_for_heartbeat | default(True) + tags: + - heartbeat + - default_cluster + - full_cluster + +- name: Deploy Cloudera Management Service + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.deployment.services.mgmt + tags: + - mgmt + - default_cluster + - full_cluster + +- name: Preload parcels from custom repo to Cloudera Manager + hosts: cloudera_manager + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.preload_parcels + when: "'custom_repo' in groups" + tags: + - preload_parcels + - default_cluster + - full_cluster + +# ENDBLOCK # Configure CM diff --git a/pvc_base_setup.yml b/pvc_base_setup.yml new file mode 100644 index 0000000..6ad4463 --- /dev/null +++ b/pvc_base_setup.yml @@ -0,0 +1,67 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution +# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run +# STARTBLOCK # Install Cluster + +- name: Deploy clusters + hosts: cloudera_manager + gather_facts: yes + roles: + - cloudera.cluster.deployment.cluster + tags: + - cluster + - default_cluster + - full_cluster + +# ENDBLOCK # Install Cluster + +# End run +### \ No newline at end of file diff --git a/pvc_base_teardown.yml b/pvc_base_teardown.yml new file mode 100644 index 0000000..e79826c --- /dev/null +++ b/pvc_base_teardown.yml @@ -0,0 +1,98 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + +# STARTBLOCK # Teardown +# Teardown CA + +- name: Teardown CA server + hosts: ca_server + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.infrastructure.ca_certs + tasks_from: clean.yml + tags: + - teardown_ca + +- name: Teardown security artifact directories + hosts: tls + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.security.tls_clean + when: "'tls' in groups" + tags: + - teardown_tls + +# Teardown Cluster + +- name: Teardown ECS + hosts: ecs_nodes + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.teardown + tasks_from: teardown_ecs.yml + vars: + cluster: "{{ definition.clusters | selectattr('type', 'defined') | selectattr('type', 'search', 'ecs') | first }}" + tags: + - teardown_ecs + +- name: Cluster Teardown Process + hosts: all + gather_facts: no + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.teardown + tags: + - teardown_cluster + +# ENDBLOCK # Teardown \ No newline at end of file diff --git a/roles/cloudera_deploy/defaults/basic_cluster.yml b/roles/cloudera_deploy/defaults/basic_cluster.yml deleted file mode 100644 index 752963b..0000000 --- a/roles/cloudera_deploy/defaults/basic_cluster.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cloudera_manager_version: 7.4.4 - -clusters: - - name: Basic Cluster - services: [HDFS, YARN, ZOOKEEPER] - repositories: - - https://archive.cloudera.com/cdh7/7.1.7.0/parcels/ - configs: - HDFS: - DATANODE: - dfs_data_dir_list: /dfs/dn - NAMENODE: - dfs_name_dir_list: /dfs/nn - SECONDARYNAMENODE: - fs_checkpoint_dir_list: /dfs/snn - YARN: - RESOURCEMANAGER: - yarn_scheduler_maximum_allocation_mb: 4096 - yarn_scheduler_maximum_allocation_vcores: 4 - NODEMANAGER: - yarn_nodemanager_resource_memory_mb: 4096 - yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm - yarn_nodemanager_log_dirs: /var/log/nm - GATEWAY: - mapred_submit_replication: 3 - mapred_reduce_tasks: 6 - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - host_templates: - Master1: - HDFS: [NAMENODE, SECONDARYNAMENODE] - YARN: [RESOURCEMANAGER, JOBHISTORY] - ZOOKEEPER: [SERVER] - Workers: - HDFS: [DATANODE] - YARN: [NODEMANAGER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true \ No newline at end of file diff --git a/roles/cloudera_deploy/defaults/main.yml b/roles/cloudera_deploy/defaults/main.yml deleted file mode 100644 index bcdebbe..0000000 --- a/roles/cloudera_deploy/defaults/main.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default Paths -default_local_temp_dir: '/tmp' -default_sshkey_path: '~/.ssh' -default_config_path: '~/.config/cloudera-deploy' - -# Default names -default_name_prefix: cldr -default_ssh_key_suffix: _ssh_rsa -default_download_mirror_prefix: cache - -# Default Basic Cluster Config -default_cluster_definition_file: "defaults/basic_cluster.yml" - -# Default Artefact Filenames -default_profile_path: "{{ [default_config_path, 'profiles'] | path_join }}" -default_profile_file: "default" -default_definition_file: "definition.yml" -default_cluster_file: "cluster.yml" -default_inventory_template: "inventory_template.ini" -default_static_inventory: "inventory_static.ini" -download_mirror_file: "{{ [default_config_path, 'download_mirror.ini'] | path_join }}" - -include_inventory_file: '' - -# Default behavior -use_default_cluster_definition: no - -# Default parcel cache -default_enable_download_mirror: no -default_parcel_distro: el7 # el8, bionic, focal -default_download_link_expiry: 3600 - -# Default Deployment Controls -default_infra_deployment_engine: ansible -default_infra_type: aws -default_infra_region: us-east-1 - -# Terraform defaults -default_terraform_base_dir: "{{ [default_config_path, 'terraform'] | path_join }}" \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/auto_terraform_state.yml b/roles/cloudera_deploy/tasks/auto_terraform_state.yml deleted file mode 100644 index 9e60f32..0000000 --- a/roles/cloudera_deploy/tasks/auto_terraform_state.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Resources for remote_s3 state storage - when: - - globals.terraform.state_storage == 'remote_s3' - block: - - # Create or Teardown the resources - - name: AWS Bucket for Remote State Storage - amazon.aws.aws_s3: - region: "{{ globals.region }}" - bucket: "{{ globals.terraform.remote_state_bucket}}" - mode: "{{ ('teardown' not in ansible_run_tags) | ternary('create', 'delete') }}" # Check ansible tag to determine action - permission: private - register: __infra_aws_storage_locations_info - - - name: AWS DynamoDB for Remote State Locking - community.aws.dynamodb_table: - region: "{{ globals.region }}" - name: "{{ globals.terraform.remote_state_lock_table }}" - read_capacity: 1 - write_capacity: 1 - hash_key_name: LockID - hash_key_type: STRING - state: "{{ ('teardown' not in ansible_run_tags) | ternary('present', 'absent') }}" # Check ansible tag to determine action - - - name: Print remote state configuration - when: "'teardown' not in ansible_run_tags" - ansible.builtin.debug: - msg: - - "Resources for remote_s3 Terraform State created." - - "S3 Bucket Name: {{ globals.terraform.remote_state_bucket}}" - - "DynamoDB Locking Table: {{ globals.terraform.remote_state_lock_table}}" - verbosity: 3 \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml b/roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml deleted file mode 100644 index a4a5a4e..0000000 --- a/roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check for a Static Inventory file - stat: - path: "{{ init__dynamic_inventory_artefact }}" - register: __inventory_static - -- name: Create a backup - when: __inventory_static.stat.exists - copy: - src: "{{ __inventory_static.stat.path }}" - dest: "{{ [init__dynamic_inventory_artefact | splitext | first, ansible_date_time.epoch] | join('.') }}" - -- name: Remove static inventory file - ansible.builtin.file: - path: "{{ init__dynamic_inventory_artefact }}" - state: absent \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml b/roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml deleted file mode 100644 index a2cde2f..0000000 --- a/roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Load Definition file - ansible.builtin.include_vars: - file: "{{ init__cluster_definition_file }}" - name: _pre_template_cluster - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label: __play_host - -- name: Set specific Facts for later use in Cluster Deployment - ansible.builtin.set_fact: - preload_parcels: "{{ download_mirror_file_list | default([]) }}" - custom_repo_rehost_files: "{{ download_mirror_file_list | default([]) }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label: __play_host - -- name: Set Sensitive Facts with no-log for later use in Cluster Deployment - ansible.builtin.set_fact: - cloudera_manager_admin_password: "{{ globals.admin_password }}" - cloudera_manager_license_file: "{{ globals.cloudera_license_file | default(omit) }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - no_log: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label: __play_host - -- name: Include Definition Vars to allow overrides to propagate to Cluster Deployment - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label : __play_host - -- name: Include Cluster Definition override - ansible.builtin.include_vars: - file: "{{ init__cluster_definition_file }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label : __play_host \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/init.yml b/roles/cloudera_deploy/tasks/init.yml deleted file mode 100644 index 27eb1ca..0000000 --- a/roles/cloudera_deploy/tasks/init.yml +++ /dev/null @@ -1,483 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Check versions -- name: Get Python packages - register: __python_env_packages - command: pip freeze - -- name: Get Ansible Collections - register: __ansible_collection_list - command: ansible-galaxy collection list - -- name: Get Ansible Roles - register: __ansible_role_list - command: ansible-galaxy role list - -- name: Log Runtime information - ansible.builtin.debug: - msg: "{{ __runtime_item }}" - loop_control: - loop_var: __runtime_item - loop: - - "Ansible Version: {{ ansible_version }}" - - "Ansible Collections: {{ __ansible_collection_list.stdout }}" - - "Ansible Roles: {{ __ansible_role_list.stdout }}" - - "Python Version: {{ ansible_python_version }}" - - "Python Packages: {{ __python_env_packages.stdout }}" - - "Runner Version: '{{ lookup('env', 'CLDR_BUILD_VER') }}'" - -# Check path -- name: Check a Definition path has been supplied - ansible.builtin.assert: - quiet: yes - that: - - definition_path is defined - - definition_path | length > 0 - fail_msg: "You must supply a 'definition_path' pointing to your artefacts directory" - -- name: Check if Definition Path '{{ definition_path }}' exists - ansible.builtin.stat: - path: "{{ definition_path }}" - register: __definition_path_stat - -- name: Assert Definition Path is a directory - ansible.builtin.assert: - quiet: yes - fail_msg: "'definition_path' does not appear to point to an existing and reachable directory'" - that: - - __definition_path_stat.stat.isdir is defined - - __definition_path_stat.stat.isdir - -# Set File Paths -- name: Set Expected File Paths - ansible.builtin.set_fact: - init__user_profile: "{{ abs_profile | default([profile_path | default(default_profile_path), profile | default(default_profile_file)] | path_join ) }}" - init__definition_file: "{{ abs_definition | default( [definition_path, definition_file | default(default_definition_file)] | path_join ) }}" - init__cluster_file: "{{ abs_cluster | default( [definition_path, cluster_file | default(default_cluster_file)] | path_join ) }}" - init__dynamic_inventory_template: "{{ abs_template | default( [definition_path, inventory_template | default(default_inventory_template)] | path_join ) }}" - init__dynamic_inventory_artefact: "{{ abs_inventory | default( [definition_path, static_inventory | default(default_static_inventory) ] | path_join ) }}" - init__download_mirror_artefact: "{{ download_mirror_file | default(download_mirror_file) }}" - -# Handle User Config -- name: Check for User Config file - register: __user_config_stat - ansible.builtin.stat: - path: "{{ init__user_profile }}" - -- name: Load User Config - when: __user_config_stat.stat.exists - ansible.builtin.include_vars: - file: "{{ __user_config_stat.stat.path }}" - -# Handle Definition File -- name: Seek Definition files in Definition Path - register: __def_file_stat - ansible.builtin.stat: - path: "{{ init__definition_file }}" - -- name: Assert that a Definition File has been provided - ansible.builtin.assert: - quiet: yes - that: __def_file_stat.stat.exists - fail_msg: "Expected to find a definition file '{{ init__definition_file }}' in Definition Path '{{ definition_path }}'" - -# User Definition files may contain lazy templating which would break if pre-merged here, therefore the cluster defaults file is kept separate -- name: Select default provided User Definition Files - ansible.builtin.set_fact: - init__user_definition_file: "{{ __def_file_stat.stat.path }}" - init__cluster_definition_file: "{{ __def_file_stat.stat.path }}" - -# Handle separate Cluster File -- name: Seek Cluster Definition files in Definition Path - register: __clus_file_stat - ansible.builtin.stat: - path: "{{ init__cluster_file }}" - -- name: Include vars to top level for other facts in Definition File - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - -# Must be included before cluster definition checks as it may have logic control switches -- name: Override with separate Cluster file if provided - when: __clus_file_stat.stat.exists - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ __clus_file_stat.stat.path }}" - -# Override with default cluster definition if requested, regardless of files found -- name: Use default cluster definition as override if requested - when: use_default_cluster_definition | bool - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ default_cluster_definition_file }}" - -- name: Include vars from User Definition File to private dict to check for Globals - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - name: __def_vars - -# Note that this depends on the earlier set_fact for globals to take precedence over include_vars -- name: Include Cluster definition file for current localhost use after User Definition is Loaded - ansible.builtin.include_vars: - file: "{{ init__cluster_definition_file }}" - -- name: If Purge is defined, check it is boolean - when: purge is defined - ansible.builtin.assert: - that: purge|bool is sameas true or purge|bool is sameas false - fail_msg: "purge key is present in definition, but not a boolean as expected" - quiet: yes - -# Admin Password -- name: Prompt User for a password if not provided in config or vault - when: admin_password is undefined or admin_password | length < 2 - block: - - name: Prompt User for Password if not supplied - no_log: true - pause: - prompt: "No admin password found in profile.yml or extra_vars, or provided password too short; please provide a Password" - register: __user_input_password - - - name: Set Admin password - no_log: true - ansible.builtin.set_fact: - admin_password: "{{ __user_input_password.user_input }}" - -- name: Assert user has supplied an Admin Password - no_log: true - ansible.builtin.assert: - quiet: yes - that: - - admin_password is defined - - admin_password | length > 2 - fail_msg: "You must supply an Admin Password of at least 2 chars" - -# Merge User Profile to Globals -- name: Marshal User Config into Globals - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(user_config , recursive=True) }}" - vars: - user_config: - name_prefix: "{{ name_prefix | default(default_name_prefix) }}" - tags: "{{ tags | default(omit) }}" - region: "{{ infra_region | default(default_infra_region) }}" - infra_deployment_engine: "{{ infra_deployment_engine | default(default_infra_deployment_engine) }}" - infra_type: "{{ infra_type | default(default_infra_type) }}" - terraform: - base_dir: "{{ terraform.base_dir | default(default_terraform_base_dir) | expanduser }}" - state_storage: "{{ terraform.state_storage | default(omit) }}" - auto_remote_state: "{{ terraform.auto_remote_state | default(False) }}" - remote_state_bucket: "{{ terraform.remote_state_bucket | default(omit) }}" - remote_state_lock_table: "{{ terraform.remote_state_lock_table | default(omit) }}" - ssh: - public_key_id: "{{ public_key_id | default(omit) }}" - public_key_file: "{{ public_key_file | default(omit) }}" - public_key_text: "{{ public_key_text | default(omit) }}" - private_key_file: "{{ private_key_file | default(omit) }}" - key_path: "{{ ssh_key_path | default(default_sshkey_path) }}" - cloudera_license_file: "{{ license_file | default(omit) }}" - gcloud_credential_file: "{{ gcloud_credential_file | default(omit) }}" - cdp_profile: "{{ cdp_profile | default(omit) }}" - cdp_region: "{{ cdp_region | default(omit) }}" - aws_profile: "{{ aws_profile | default(omit) }}" - force_teardown: "{{ purge | default(omit) }}" - env_vars: "{{ env_vars | default(omit) }}" - -- name: Merge overwrite globals from Definition file with Globals on User File - when: __def_vars.globals is defined - ansible.builtin.set_fact: - globals: "{{ globals | combine(__def_vars.globals, recursive=True) }}" - -# Validate Name Prefix -- name: Check supplied Namespace (Azure) - when: - - globals.infra_type == 'azure' - - "'teardown' not in {{ ansible_run_tags }}" - ansible.builtin.assert: - that: - - globals.name_prefix | length > 1 - - globals.name_prefix | length < 5 - - globals.name_prefix | regex_search('^[a-zA-Z]') - fail_msg: "You must supply a valid Namespace" - quiet: yes - -- name: Check supplied Namespace - when: - - globals.infra_type != 'azure' - - "'teardown' not in {{ ansible_run_tags }}" - ansible.builtin.assert: - that: - - globals.name_prefix | length > 1 - - globals.name_prefix | length < 8 - - globals.name_prefix | regex_search('^[a-zA-Z]') - fail_msg: "You must supply a valid Namespace" - quiet: yes - -- name: Check Deployment Engine variable - ansible.builtin.assert: - that: - - globals.infra_deployment_engine in ['ansible', 'terraform'] - fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'" - -- name: Check Supplied terraform_base_dir variable - when: - - globals.infra_deployment_engine == 'terraform' - ansible.builtin.assert: - that: - - globals.terraform.base_dir is defined - - globals.terraform.base_dir | length > 0 - fail_msg: "You must supply a 'terraform_base_dir' where Terraform assets will be placed" - quiet: yes - -- name: Check Supplied terraform_auto_remote_state variable - when: - - globals.infra_deployment_engine == 'terraform' - ansible.builtin.assert: - that: - - (globals.terraform.auto_remote_state|bool is sameas true) or (globals.terraform.auto_remote_state|bool is sameas false) - fail_msg: "The terraform.auto_remote_state variable must be a boolean variable" - quiet: yes - -# SSH -- name: Use default SSH public key id - when: globals.ssh.public_key_id is undefined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__default_ssh_key_id, recursive=True) }}" - vars: - __default_ssh_key_id: - ssh: - public_key_id: "{{ globals.name_prefix }}" - -- name: Generate SSH public and private keys - when: (globals.ssh.public_key_file is undefined or globals.ssh.public_key_file | length < 3) and globals.ssh.public_key_text is undefined - block: - - name: Generate a SSH keypair (public and private keys) - register: __generated_ssh_keys - community.crypto.openssh_keypair: - path: "{{ [default_sshkey_path, __generated_keypair_name] | path_join }}" - comment: "{{ globals.name_prefix }} (auto-generated)" - type: rsa - size: 4096 - regenerate: never - force: no - vars: - __generated_keypair_name: "{{ globals.name_prefix + default_ssh_key_suffix }}" - - - name: Set facts for the generated SSH keypair details - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__generated_keypair, recursive=True) }}" - vars: - __generated_keypair: - ssh: - private_key_file: "{{ __generated_ssh_keys.filename }}" - public_key_file: "{{ [ __generated_ssh_keys.filename, 'pub' ] | join('.') }}" - -- name: Load SSH public key file to text - when: globals.ssh.public_key_file is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__public_key_globals , recursive=True) }}" - vars: - __public_key_globals: - ssh: - public_key_text: "{{ lookup('file', globals.ssh.public_key_file ) | default(omit) }}" - -- name: Validate SSH Private Key File has acceptable permissions - when: globals.ssh.private_key_file is defined - block: - - name: Get information for SSH Private Key File - ansible.builtin.stat: - path: "{{ globals.ssh.private_key_file }}" - register: __private_key_file_stat - - - name: Assert that SSH Private Key has valid permissions - ansible.builtin.assert: - that: - - __private_key_file_stat.stat.mode == '0400' or __private_key_file_stat.stat.mode == '0600' - fail_msg: - - "SSH Private Key at {{ __private_key_file_stat.stat.path }} has invalid permissions" - - "Permissions are {{ __private_key_file_stat.stat.mode }}" - - "Permissions should be 0400 or 0600" - -# Parcel Distro -- name: Determine preferred Parcel Distribution - ansible.builtin.set_fact: - init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}" - -# Read in Dynamic Inventory -- name: Seek Inventory Template in Definition Path - register: __di_template_stat - ansible.builtin.stat: - path: "{{ init__dynamic_inventory_template }}" - -# inventory_dir is not defined when a user passes in an inventory with -i, so it is a useful check -# No point loading Dynamic Inventory if we are not doing infrastructure in this run -- name: Handle Dynamic Inventory Template - when: - - inventory_dir is defined - - __di_template_stat.stat.exists - block: - - name: Load in Dynamic Inventory Template - include_tasks: refresh_inventory.yml - vars: - include_inventory_file: "{{ __di_template_stat.stat.path }}" - - - name: Print Dynamic Inventory groups to debug at Verbosity 3 - debug: - msg: "{{ groups }}" - verbosity: 3 - - - name: Check expected minimum host groups appear in Inventory - ansible.builtin.assert: - quiet: yes - that: - - groups.cluster is defined - - groups.cloudera_manager is defined - fail_msg: "Parsed Inventory Template did not contain minimum expected groups for a Cloudera Cluster deployment" - - - name: Check a custom_repo is part of the cluster definition if using Download Mirror - when: use_download_mirror | default(default_enable_download_mirror) | bool - ansible.builtin.assert: - that: "'custom_repo' in groups" - fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" - - - name: Extract list of hosts from Dynamic Inventory Template - when: groups.all | length > 0 - ansible.builtin.set_fact: - __dynamic_inventory_host_list: "{{ groups.all | difference(['localhost']) }}" - - - name: Set Dynamic Inventory host count in Globals - when: __dynamic_inventory_host_list | length > 0 - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __di_entry | default(omit), recursive=True ) }}" - loop_control: - loop_var: __di_entry - loop: - - dynamic_inventory: - vm: - count: "{{ __dynamic_inventory_host_list | count }}" - os: "{{ init__parcel_distro }}" - always: - - name: Remove Dynamic Inventory Template from current inventory - include_tasks: refresh_inventory.yml - -- name: Add no_log variables to globals - no_log: true - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__no_log_globals, recursive=True) }}" - vars: - __no_log_globals: - admin_password: "{{ admin_password | mandatory }}" - -- name: Set CM facts - ansible.builtin.set_fact: - cloudera_manager_license_file: "{{ globals.cloudera_license_file | default(omit) }}" - cloudera_manager_admin_password: "{{ globals.admin_password }}" - no_log: true - -# Download Mirror -- name: Prepare for Download Mirror Population if requested and necessary - when: - - use_download_mirror | default(default_enable_download_mirror) | bool - - "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: prepare_download_mirror - -# Ansible tempfile doesn't appear to work well on ansible-runner -- name: Set local Temp directory if not supplied - ansible.builtin.set_fact: - local_temp_dir: "{{ local_temp_dir | default(default_local_temp_dir) }}" - -- name: Set GCloud Environment Variables if needed - when: globals.gcloud_credential_file is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_gcp_entries, recursive=True ) }}" - vars: - env_gcp_entries: - env_vars: - GCP_AUTH_KIND: serviceaccount - GCP_SERVICE_ACCOUNT_FILE: "{{ globals.gcloud_credential_file }}" - -- name: Set Profile Env Vars if required - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_var_entries, recursive=True ) }}" - vars: - env_var_entries: - env_vars: - CDP_PROFILE: "{{ globals.cdp_profile | default(omit) }}" - AWS_PROFILE: "{{ globals.aws_profile | default(omit) }}" - AWS_REGION: "{{ globals.region | default(omit) }}" - -- name: Print globals to debug at end of init (verbosity 3) - ansible.builtin.debug: - msg: "{{ globals }}" - verbosity: 3 - -- name: Determine if Cloud Roles should be called - ansible.builtin.set_fact: - init__call_cloud_role: "{{ infra is defined or env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" - init__call_cdp_public: "{{ env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" - -- name: Check Admin Password is CDP Public compliant when calling CDP Public - when: init__call_cdp_public | bool - ansible.builtin.assert: - that: - - admin_password is match('^(?=.*[A-Za-z])(?=.*\\d)(?=.*[@$!%*#?&])[A-Za-z\\d@$!%*#?&]{8,64}$') - fail_msg: >- - Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars. - quiet: yes - -# Provision Terraform remote state resources if requested -- name: Initialize Terraform remote state resources - when: - - init__call_cloud_role | bool - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - block: - # Set resource variable names if not already done - - name: Set variables for remote state bucket if not set - when: (globals.terraform.remote_state_bucket is not defined) or - ( (globals.terraform.remote_state_bucket) | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}" - - - name: Set variables for remote state lock table if not set - when: (globals.terraform.remote_state_lock_table is not defined) or - (globals.terraform.remote_state_lock_table | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}" - - - name: Create remote state resources - when: "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: auto_terraform_state - -- name: Prepare for Download Mirror Population if requested and necessary - when: - - use_download_mirror | default(default_enable_download_mirror) | bool - - "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: prepare_download_mirror \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/inject_download_mirror.yml b/roles/cloudera_deploy/tasks/inject_download_mirror.yml deleted file mode 100644 index f087f5b..0000000 --- a/roles/cloudera_deploy/tasks/inject_download_mirror.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check if Download Mirror cache file exists - register: __download_mirror_file_stat - ansible.builtin.stat: - path: "{{ init__download_mirror_artefact }}" - -- name: Prepare for Download Mirror parsing - when: - - __download_mirror_file_stat.stat.exists - - use_download_mirror | default(default_enable_download_mirror) | bool - block: - - name: Check a custom_repo is part of the cluster definition if using Download Mirror - when: use_download_mirror | default(default_enable_download_mirror) | bool - ansible.builtin.assert: - that: "'custom_repo' in groups" - fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" - - - name: Handle AWS cache key generation - when: globals.infra_type == 'aws' - block: - - name: Get AWS Account Info - amazon.aws.aws_caller_info: - register: __aws_caller_info - - - name: Set parcel cache ini lookup key - ansible.builtin.set_fact: - init__download_mirror_ini_key: "{{ __aws_caller_info.account }}" - -# Ini lookup fails when section is not present, but lacks good control characteristics that I can find -- name: Determine if there is a relevant cache entry - when: - - init__download_mirror_ini_key is defined - - use_download_mirror | default(default_enable_download_mirror) | bool - ignore_errors: yes - ansible.builtin.set_fact: - __download_mirror_ini_entry: "{{ lookup('ini', __ini_lookup) }}" - vars: - __ini_lookup: ".+{{ init__download_mirror_ini_key }}.+ section={{ globals.infra_type }}:{{ globals.region }} file={{ init__download_mirror_artefact }} re=yes" - -- name: Handle Download Mirror injection if cache entry found - when: - - __download_mirror_ini_entry is defined - block: - - name: Generate a unique name - set_fact: - __tmp_cluster_file: "{{ ['/tmp', 99999999 | random | to_uuid] | path_join }}" - - - name: Copy Cluster definition to temp file - copy: - src: "{{ init__cluster_definition_file }}" - dest: "{{ __tmp_cluster_file }}" - - - name: Inject Parcel cache entries to Repository URLs - ansible.builtin.replace: - name: "{{ __tmp_cluster_file }}" - regexp: '^(\s+\-\s)https://archive\.cloudera\.com(\/.+)$' - replace: '\1http://{{ groups.custom_repo | first }}\2' - - - name: Set Cluster Definition file to Temp file with parcel cache entries - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ __tmp_cluster_file }}" - - - name: Fetch repositories from cluster definition - ansible.builtin.set_fact: - init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}" - - - name: Prepare lookup list of Repository entries - loop: "{{ init__preparse_repo_listing }}" - loop_control: - loop_var: __cluster_repo_item - ansible.builtin.set_fact: - init__cluster_repo_entries: "{{ init__cluster_repo_entries | default([]) + [__cluster_repo_item | urlsplit('path') ] }}" - - - name: Create list of Download Mirror URLs filtered to required repositories and distros - loop: "{{ init__cluster_repo_entries }}" - loop_control: - loop_var: __cluster_repo_path_item - ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __download_mirror_ini_entry - | select('search', __cluster_repo_path_item) - | select('search', init__parcel_distro) - | list }}" - - - name: Ensure manifest is included in Download Mirror URLs if present - loop: "{{ init__cluster_repo_entries }}" - loop_control: - loop_var: __cluster_repo_path_item - ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __download_mirror_ini_entry - | select('search', 'manifest.json') - | list }}" - -- name: Get AWS Specific download URIs - when: - - globals.infra_type == 'aws' - - init__urls_to_sign is defined - block: - - name: Generate signed URIs for hosted parcels to be pulled into custom_repo - register: __s3_signed_uris - loop: "{{ init__urls_to_sign }}" - loop_control: - loop_var: __s3_bucket_uri - amazon.aws.aws_s3: - bucket: "{{ __s3_bucket_uri | regex_replace('^.+//(.+)\\.s3.+$', '\\1') }}" - object: "{{ __s3_bucket_uri | regex_replace('^.+amazonaws\\.com(.+)$', '\\1') }}" - ignore_nonexistent_bucket: yes - expiry: "{{ download_link_expiry | default(default_download_link_expiry) }}" - mode: geturl - - - name: Set List of files to download to custom_repo - ansible.builtin.set_fact: - download_mirror_file_list: "{{ __s3_signed_uris.results | json_query('[*].url') | list }}" diff --git a/roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml b/roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml deleted file mode 100644 index c310169..0000000 --- a/roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Generate a unique name - ansible.builtin.set_fact: - __tmp_dynamic_inventory_artefact: "{{ [inventory_dir, 99999999 | random | to_uuid] | path_join }}" - -- name: Copy Host Inventory Template to Temporary Static Inventory Artefact - ansible.builtin.copy: - src: "{{ init__dynamic_inventory_template }}" - dest: "{{ __tmp_dynamic_inventory_artefact }}" - -- name: Update Static Inventory Artefact with Host Entries from Cloud Infrastructure Run - loop: "{{ __dynamic_inventory_host_list | zip( infra__dynamic_inventory_host_entries ) }}" - loop_control: - loop_var: __infra_inventory_compute_item - ansible.builtin.replace: - name: "{{ __tmp_dynamic_inventory_artefact }}" - regexp: '(\s+){{ __infra_inventory_compute_item.0 }}(\s+)' - replace: '\1{{ __infra_inventory_compute_item.1 }}\2' - -- name: Stat the Temporary Artefact - ansible.builtin.stat: - path: "{{ __tmp_dynamic_inventory_artefact }}" - register: __tmp_inventory_static - -- name: Check for an existing Dynamic Inventory Artefact file - ansible.builtin.stat: - path: "{{ init__dynamic_inventory_artefact }}" - register: __inventory_static - -- name: Create a backup if the files are different - when: - - __inventory_static.stat.exists - - __tmp_inventory_static.stat.checksum != __inventory_static.stat.checksum - ansible.builtin.copy: - src: "{{ __inventory_static.stat.path }}" - dest: "{{ [init__dynamic_inventory_artefact | splitext | first, ansible_date_time.epoch] | join('.') }}" - -- name: Copy Temporary Dynamic Inventory Artefact to Inventory Artefact - ansible.builtin.copy: - src: "{{ __tmp_inventory_static.stat.path }}" - dest: "{{ init__dynamic_inventory_artefact }}" - force: yes - -- name: Remove Temporary Artefact file - ansible.builtin.file: - path: "{{ __tmp_inventory_static.stat.path }}" - state: absent diff --git a/roles/cloudera_deploy/tasks/populate_download_mirror.yml b/roles/cloudera_deploy/tasks/populate_download_mirror.yml deleted file mode 100644 index c2681a5..0000000 --- a/roles/cloudera_deploy/tasks/populate_download_mirror.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Ensure paths exist for File downloads to match required object structure [ will retry while utility VM boots ] - loop: "{{ globals.download_mirror_targets }}" - loop_control: - loop_var: __tmp_mirror_dir_item - ansible.builtin.file: - path: "/tmp/{{ globals.utility_bucket_name }}{{ __tmp_mirror_dir_item | urlsplit('path') | dirname }}" - state: directory - -- name: Request Async Download of Files to path structure - when: globals.download_mirror_targets - register: __download_mirror_rehost_results - loop: "{{ globals.download_mirror_targets }}" - loop_control: - loop_var: __mirror_fetch_item - async: 7200 - poll: 0 - ansible.builtin.get_url: - url: "{{ __mirror_fetch_item }}" - dest: "/tmp/{{ globals.utility_bucket_name }}{{ __mirror_fetch_item | urlsplit('path') }}" - url_username: "{{ cloudera_manager_repo_username | default(omit) }}" - url_password: "{{ cloudera_manager_repo_password | default(omit) }}" - -- name: Track async downloads to completion [ This may take up to an hour the first time for multi-gb Parcels ] - loop: "{{ __download_mirror_rehost_results.results }}" - loop_control: - loop_var: __download_async_item - register: __async_download_results - until: __async_download_results.finished is defined and __async_download_results.finished - delay: 30 - retries: 240 - async_status: - jid: "{{ __download_async_item.ansible_job_id }}" - failed_when: - - __download_async_item.failed == True - - __download_async_item.finished != 1 - -# Unpack parcel tarballs -- name: Unpack parcel tarballs for convenient use - loop: "{{ globals.download_mirror_targets | select('search', 'parcels.tar.gz') | list }}" - loop_control: - loop_var: __parcel_unpack_item - ansible.builtin.unarchive: - extra_opts: [ --strip-components=1 ] - remote_src: yes - src: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') }}" - dest: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') | dirname }}" - keep_newer: yes - -- name: Upload Download Mirror for AWS to S3 - when: globals.infra_type == 'aws' - block: - # Prepare to sync cache dir to S3 - - name: Setup System Rhel8 - when: - - ansible_os_family == 'RedHat' - - ansible_distribution_major_version | int >= 8 - become: yes - ansible.builtin.package: - lock_timeout: 180 - name: "{{ __package_item }}" - update_cache: yes - state: present - loop_control: - loop_var: __package_item - loop: - - epel-release - - python3 - - - name: Setup system Rhel7 - when: - - ansible_os_family == 'RedHat' - - ansible_distribution_major_version | int < 8 - become: yes - ansible.builtin.package: - name: "{{ __package_item }}" - state: present - lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(180, omit) }}" - loop_control: - loop_var: __package_item - loop: - - epel-release - - python-pip - - - name: Setup system Debian - when: ansible_os_family == "Debian" - block: - - name: enable Debian Repos - become: yes - apt_repository: - repo: "{{ __repo_item }}" - loop_control: - loop_var: __repo_item - loop: - - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }} universe" - - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-updates universe" - - "deb http://security.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-security universe" - - - name: Install Pip on Debian - become: yes - ansible.builtin.apt: - update_cache: yes - name: python3-pip - state: present - - - name: Prepare host for s3 actions - become: yes - ansible.builtin.pip: - name: "{{ __pip_item }}" - loop_control: - loop_var: __pip_item - loop: - - futures - - "{{ (ansible_python_version[0] == '2') | ternary('boto3 >= 1.4.4,<1.18', 'boto3 >= 1.20.0') }}" - - - name: Sync downloaded Files paths to S3 cache bucket - become: yes - community.aws.s3_sync: - bucket: "{{ globals.utility_bucket_name }}" - file_root: "/tmp/{{ globals.utility_bucket_name }}" - permission: private diff --git a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml b/roles/cloudera_deploy/tasks/prepare_download_mirror.yml deleted file mode 100644 index 10ef568..0000000 --- a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Read in cluster definition without jinja parsing -- name: Fetch repositories from cluster definition - ansible.builtin.set_fact: - init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}" - -- name: Check that a Cloudera License is presented if mirroring from files behind Cloudera Subscription - when: init__preparse_repo_listing | select('search', 'archive.cloudera.com/p') | length > 0 - ansible.builtin.assert: - that: - - globals.cloudera_license_file is defined - - globals.cloudera_license_file | length > 0 - fail_msg: "You must Supply a Cloudera License file to download and mirror files from archive.cloudera.com" - -- name: Separate direct Repos from tarballs and set Initial Facts - ansible.builtin.set_fact: - __init_parcel_repos: "{{ init__preparse_repo_listing | reject('search', 'tar.gz') | default([]) }}" - __init_tarball_links: "{{ init__preparse_repo_listing | select('search', 'tar.gz') | default([]) }}" - -# This sets 'manifests' on the calling host, and provides the repo login details -- name: Get Parcel Manifests - ansible.builtin.include_role: - name: cloudera.cluster.deployment.repometa - public: yes - vars: - repositories: "{{ __init_parcel_repos }}" - cluster_os_distribution: "{{ init__parcel_distro }}" - -- name: Extract Parcel URLs from Manifests - ansible.builtin.set_fact: - __parcel_urls: "{{ manifests.results | cloudera.cluster.extract_parcel_urls }}" - __parcel_distro_search_term: "{{ init__parcel_distro }}.parcel" - -- name: Filter Parcels by distro - ansible.builtin.set_fact: - __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}" - -- name: Prepare target Download Mirror listing with parcels and attendant files - when: __filtered_parcel_urls | length > 0 - loop: "{{ __filtered_parcel_urls }}" - loop_control: - loop_var: __filtered_parcel_item - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets | default(__init_tarball_links) + [__filtered_parcel_item, __filtered_parcel_item + '.sha1', __filtered_parcel_item.replace(__filtered_parcel_item | basename, 'manifest.json') ] }}" - -# This sets 'cloudera_manager_repo_url' on the calling host -- name: Determine Cloudera Manager Repo - ansible.builtin.include_role: - role: cloudera.cluster.cloudera_manager.repo - vars: - install_repo_on_host: no - clusters: [] - -- name: Add Cloudera Manager Repo to File Mirror list - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets + [cloudera_manager_repo_url | regex_replace('^(.+\\/(\\d\\.\\d\\.\\d)\\/)(\\w+)\\/.+$', '\\1' + 'repo-as-tarball/cm' + '\\2' + '-' + '\\3' + '.tar.gz')] }}" - -- name: Include CSDs if set - when: - - cloudera_manager_csds is defined - - cloudera_manager_csds | length > 0 - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets + cloudera_manager_csds }}" - -- name: Resolve Download Mirror for AWS - when: globals.infra_type == 'aws' - block: - - name: Get AWS Account Info - amazon.aws.aws_caller_info: - register: __aws_caller_info - - - name: Prepare Localised Download Mirror utility bucket name - ansible.builtin.set_fact: - init__download_mirror_bucket_name: "{{ utility_bucket_name | default([ download_mirror_prefix | default(default_download_mirror_prefix), __aws_caller_info.account, globals.region ] | join('-') ) }}" - - - name: List current target cache contents if any exist - register: __download_mirror_lookup_initial - failed_when: - - __download_mirror_lookup_initial.s3_keys is not defined - - "'cannot be found' not in __download_mirror_lookup_initial.msg" - amazon.aws.aws_s3: - bucket: "{{ init__download_mirror_bucket_name }}" - mode: list - - - name: Filter Files not already in mirror to be downloaded - when: __download_mirror_lookup_initial.s3_keys is defined - loop: "{{ __download_mirror_lookup_initial.s3_keys }}" - loop_control: - loop_var: __init_s3key_item - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets | reject('match', '^.+' + __init_s3key_item + '$') | list }}" - -- name: Set Download Mirror details in Globals for Cloud run - when: - - init__file_mirror_targets | length > 0 - - init__download_mirror_bucket_name is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __download_mirror_spec, recursive=True ) }}" - vars: - __download_mirror_spec: - download_mirror_targets: "{{ init__file_mirror_targets }}" - utility_bucket_name: "{{ init__download_mirror_bucket_name }}" - create_utility_service: yes diff --git a/roles/cloudera_deploy/tasks/refresh_inventory.yml b/roles/cloudera_deploy/tasks/refresh_inventory.yml deleted file mode 100644 index 5313d2e..0000000 --- a/roles/cloudera_deploy/tasks/refresh_inventory.yml +++ /dev/null @@ -1,51 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check for additional inventory file - ansible.builtin.stat: - path: "{{ include_inventory_file }}" - register: __add_inventory_static - -- name: Generate a unique name - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.set_fact: - __tmp_add_inventory_file: "{{ [inventory_dir, 99999999 | random | to_uuid] | path_join }}" - -- name: Temporarily copy Additional Inventory to Ansible inventory dir {{ inventory_dir }} - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.copy: - src: "{{ __add_inventory_static.stat.path }}" - dest: "{{ __tmp_add_inventory_file }}" - -- name: Refresh inventory - meta: refresh_inventory - -- name: Print updated inventory to log - ansible.builtin.debug: - msg: "{{ groups }}" - verbosity: 3 - -- name: Remove temporary static inventory file - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.file: - path: "{{ __tmp_add_inventory_file }}" - state: absent From cd040d2d4b7761f6a5ab364b894b7866e35625f1 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Wed, 8 Mar 2023 16:36:59 -0500 Subject: [PATCH 2/4] Update installation order and targeted hosts (#97) * Move RDBMS role after TLS cert distribution role Signed-off-by: Webster Mudge * Update hosts for CM configuration and deployment Update conditionals for installation of Kerberos Signed-off-by: Webster Mudge * Add gather_facts to the definition verification play Signed-off-by: Webster Mudge Signed-off-by: Webster Mudge --- pvc_base_prereqs_ext.yml | 30 +++++++++++++++--------------- pvc_base_prereqs_int.yml | 6 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pvc_base_prereqs_ext.yml b/pvc_base_prereqs_ext.yml index b41284a..045be4b 100644 --- a/pvc_base_prereqs_ext.yml +++ b/pvc_base_prereqs_ext.yml @@ -63,7 +63,7 @@ - name: Verify definition [verify_definition] hosts: cloudera_manager - gather_facts: no + gather_facts: yes roles: - cloudera.cluster.verify.definition tags: @@ -86,7 +86,7 @@ # Moved from verify_parcels to reduce duplication - name: Verify definition [verify_parcels_and_roles] hosts: cloudera_manager - gather_facts: no + gather_facts: yes roles: - cloudera.cluster.verify.parcels_and_roles tags: @@ -190,19 +190,6 @@ - full_cluster # ENDBLOCK # Prepare Nodes -# STARTBLOCK # Install Cluster Service Infrastructure II - -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - cloudera.cluster.infrastructure.rdbms - tags: - - database - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cluster Service Infrastructure II # STARTBLOCK # Create Cluster Service Infrastructure - name: Install Kerberos Server @@ -300,3 +287,16 @@ and not (autotls | default(False)) # ENDBLOCK # NiFi TLS +# STARTBLOCK # Install Cluster Service Infrastructure II + +- name: Install RDBMS + hosts: db_server + become: yes + roles: + - cloudera.cluster.infrastructure.rdbms + tags: + - database + - default_cluster + - full_cluster + +# ENDBLOCK # Install Cluster Service Infrastructure II diff --git a/pvc_base_prereqs_int.yml b/pvc_base_prereqs_int.yml index 01833bc..13fe1d4 100644 --- a/pvc_base_prereqs_int.yml +++ b/pvc_base_prereqs_int.yml @@ -118,7 +118,7 @@ - full_cluster - name: Configure Cloudera Manager server - hosts: localhost + hosts: cloudera_manager gather_facts: no roles: - cloudera.cluster.cloudera_manager.config @@ -178,7 +178,7 @@ become: yes roles: - role: cloudera.cluster.prereqs.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups + when: "'krb5_server' in groups" # Only set up if cloudera.cluster has itself provisioned Kerberos tags: - kerberos - prereqs @@ -223,7 +223,7 @@ - full_cluster - name: Deploy Cloudera Management Service - hosts: localhost + hosts: cloudera_manager gather_facts: no roles: - cloudera.cluster.deployment.services.mgmt From 0531668f5154032155ea3b8d5bbccff8def6c1a0 Mon Sep 17 00:00:00 2001 From: Dan Chaffelson Date: Wed, 8 Mar 2023 16:37:00 -0500 Subject: [PATCH 3/4] Fix teardown when dynamic infra not reachable (#93) * Remove Infra availability checks during teardown as Infra may not actually be reachable * Add 'pvc' tag to cluster deployment for the scenario where ECS is being added to an existing CDP Base cluster * Remove duplicate call to deploy local OS user accounts for ECS Signed-off-by: Daniel Chaffelson --- pvc_base_prereqs_ext.yml | 6 ------ pvc_base_setup.yml | 1 + pvc_base_teardown.yml | 13 ------------- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/pvc_base_prereqs_ext.yml b/pvc_base_prereqs_ext.yml index 045be4b..8d18044 100644 --- a/pvc_base_prereqs_ext.yml +++ b/pvc_base_prereqs_ext.yml @@ -122,10 +122,6 @@ ansible.builtin.include_role: name: cloudera.cluster.prereqs.pvc_ecs - - name: Create user accounts for ECS nodes - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.user_accounts_ecs - - name: Create local user accounts hosts: cloudera_manager, cluster become: yes @@ -150,8 +146,6 @@ ansible.builtin.include_role: name: cloudera.cluster.prereqs.user_accounts_ecs -# TODO: add pvc tag to all ecs_nodes plays - - name: Install JDK hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes become: yes diff --git a/pvc_base_setup.yml b/pvc_base_setup.yml index 6ad4463..bcd2567 100644 --- a/pvc_base_setup.yml +++ b/pvc_base_setup.yml @@ -58,6 +58,7 @@ - cloudera.cluster.deployment.cluster tags: - cluster + - pvc - default_cluster - full_cluster diff --git a/pvc_base_teardown.yml b/pvc_base_teardown.yml index e79826c..7e997eb 100644 --- a/pvc_base_teardown.yml +++ b/pvc_base_teardown.yml @@ -31,19 +31,6 @@ tasks_from: prep_pvc.yml tags: - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always # ENDBLOCK # Init run # STARTBLOCK # Teardown From b3762e4532b05343a28ada50763726be1a783546 Mon Sep 17 00:00:00 2001 From: Dan Chaffelson Date: Wed, 8 Mar 2023 16:37:00 -0500 Subject: [PATCH 4/4] Remove legacy centos7-init.sh script (#98) Fixes https://github.com/cloudera-labs/cloudera-deploy/issues/70; deprecate centos7-init.sh in favour of local setup playbook in cldr-runner, as this is a better home for this setup Signed-off-by: Daniel Chaffelson --- centos7-init.sh | 78 ------------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 centos7-init.sh diff --git a/centos7-init.sh b/centos7-init.sh deleted file mode 100644 index 21d5918..0000000 --- a/centos7-init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script sets up and environment suitable for running Ansible playbooks using Centos 7. It is distinct from the docker version of runner. -# If you set any of the environment variables KUBECTL, AWS, GCLOUD, AZURE it will also install the relevant client libraries and dependencies for those operating modes. - -sudo yum install -y python36-devel git curl which bash gcc sshpass - -# 21 or later required otherwise ansible fails because of setuptools_rust. Higher than 21.0.1 gives warnings due to https://github.com/pypa/pip/issues/5599 -sudo python3 -m pip install pip==21.0.1 - -pip3 install setuptools -pip3 install ansible - -ansible-galaxy collection install --force git+https://github.com/cloudera-labs/cloudera.cluster.git -ansible-galaxy collection install --force git+https://github.com/cloudera-labs/cloudera.cloud.git -ansible-galaxy collection install --force git+https://github.com/cloudera-labs/cloudera.exe.git - -git clone https://github.com/cloudera-labs/cldr-runner runner -if [[ -z "$KUBECTL" ]]; then echo KUBECTL not requested; else sudo yum install -y kubernetes-client; fi -if [[ -z "$AWS" ]]; then echo AWS not requested; else pip3 install --no-cache-dir -r runner/payload/deps/python_aws.txt; fi -if [[ -z "$GCLOUD" ]]; then echo GCLOUD not requested; else sudo tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM -[google-cloud-sdk] -name=Google Cloud SDK -baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=0 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOM -sudo yum install -y google-cloud-sdk; pip3 install --no-cache-dir -r runner/payload/deps/python_gcp.txt; fi -if [[ -z "$AZURE" ]]; then echo AZURE not requested; else sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc; echo -e "[azure-cli] -name=Azure CLI -baseurl=https://packages.microsoft.com/yumrepos/azure-cli -enabled=1 -gpgcheck=1 -gpgkey=https://packages.microsoft.com/keys/microsoft.asc" | sudo tee /etc/yum.repos.d/azure-cli.repo; sudo yum install -y azure-cli; pip3 install --no-cache-dir -r runner/payload/deps/python_azure.txt; fi -if [[ -z "$CDPY" ]]; then echo CDPY not requested; else pip3 install git+git://github.com/cloudera-labs/cdpy@main#egg=cdpy --upgrade; fi - -ansible-galaxy install -r runner/payload/deps/ansible.yml -pip3 install -r runner/payload/deps/python_base.txt - -tee -a ansible.cfg << EOF -[defaults] -inventory=inventory -callback_whitelist = ansible.posix.profile_tasks -host_key_checking = False -gathering = smart -pipelining = True -deprecation_warnings=False -[ssh_connection] -retries = 10 -EOF - -echo "Example command: " -echo 'export ANSIBLE_LOG_PATH=~/ansible.log; ansible-playbook main.yml -e "definition_path=examples/sandbox" --ask-pass -vv -i examples/sandbox/inventory_static.ini' - -cat <