diff --git a/README.md b/README.md index 1868ad7f..d365bdd4 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,15 @@ The collection has several dependencies that should resolve automatically via th - [cloudera.cloud](https://github.com/cloudera-labs/cloudera.cloud.git) (on Cloudera Labs) - [ansible.netcommon](https://github.com/ansible-collections/ansible.netcommon) - [community.general](https://github.com/ansible-collections/community.general) -- [community.aws](https://github.com/ansible-collections/community.aws) -- [amazon.aws](https://github.com/ansible-collections/amazon.aws) -- [azure.azcollection](https://github.com/ansible-collections/azure) -- [google.cloud](https://github.com/ansible-collections/google.cloud) -- [netapp.azure](https://github.com/ansible-collections/netapp.azure) + +You may want to install additional cloud provider collections depending on your target platform: + +| Cloud Provider | Dependency | Version | +| Azure | [azure.azcollection](https://github.com/ansible-collections/azure) | `1.11.0` | +|| [netapp.azure](https://github.com/ansible-collections/netapp.azure) | `21.10.0` | +| AWS | [amazon.aws](https://github.com/ansible-collections/amazon.aws) | `3.0.0` | +|| [community.aws](https://github.com/ansible-collections/community.aws) | `3.0.1` | +| GCP | [google.cloud](https://github.com/ansible-collections/google.cloud) | `1.0.2` | ## Option #1: Install from GitHub @@ -74,6 +78,9 @@ The collection requires Ansible `2.10.0` or higher. + `cloudera.cloud` + `ansible.netcommon` + `community.general` + +You will need to add the following, depending on your target deployment: + + `community.aws` + `amazon.aws` + `azure.azcollection` diff --git a/galaxy.yml b/galaxy.yml index d26c2146..ff8a571f 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -16,34 +16,28 @@ namespace: cloudera name: exe -version: 1.7.5 +version: 2.0.0-rc1 readme: README.md authors: - Webster Mudge -- Daniel Chaffelson +- Daniel Chaffelson description: > A set of roles and other artifacts that encapsulate best practices and opinionated deployment and management processes for Cloudera Data Platform (CDP) Public Cloud - and Private Cloud, including Private Cloud Base, deployments. + and Private Cloud deployments. license_file: LICENSE tags: - cloudera - cdp - azure -- ec2 +- aws - gcp -- openstack dependencies: 'git+https://github.com/cloudera-labs/cloudera.cloud.git': 'main' 'ansible.netcommon': '2.5.1' - 'community.aws': '3.0.1' 'community.general': '4.5.0' - 'amazon.aws': '3.0.0' - 'azure.azcollection': '1.11.0' - 'google.cloud': '1.0.2' - 'netapp.azure': '21.10.0' repository: https://github.com/cloudera-labs/cloudera.exe # TODO: Set documentation URL diff --git a/playbooks/pbc_infra_setup.yml b/playbooks/pbc_infra_setup.yml new file mode 100644 index 00000000..a5e1bc57 --- /dev/null +++ b/playbooks/pbc_infra_setup.yml @@ -0,0 +1,44 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + hosts: "{{ target | default('localhost') }}" + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + - name: Validate CDP Public Cloud infrastructure configuration + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: validate + tags: + - validate + - initialize + - infra + + - name: Initialize CDP Public Cloud infrastructure setup + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: initialize_setup + tags: + - initialize + - infra + + - name: Set up CDP Public Cloud infrastructure + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: setup + tags: + - infra diff --git a/playbooks/pbc_infra_teardown.yml b/playbooks/pbc_infra_teardown.yml new file mode 100644 index 00000000..040eff73 --- /dev/null +++ b/playbooks/pbc_infra_teardown.yml @@ -0,0 +1,45 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + hosts: "{{ target | default('localhost') }}" + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + - name: Validate CDP Public Cloud infrastructure configuration + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: validate + tags: + - validate + - initialize + - infra + + - name: Initialize CDP Public Cloud infrastructure teardown + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: initialize_teardown + tags: + - initialize + - infra + + - name: Tear down CDP Public Cloud infrastructure + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: teardown + tags: + - infra + diff --git a/playbooks/pbc_setup.yml b/playbooks/pbc_setup.yml new file mode 100644 index 00000000..80ca131f --- /dev/null +++ b/playbooks/pbc_setup.yml @@ -0,0 +1,76 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up CDP Public Cloud + hosts: "{{ target | default('localhost') }}" + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Validate Platform configuration + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: validate + tags: + - validate + - initialize + - plat + - run + + - name: Validate Data Services configuration + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: validate + tags: + - validate + - initialize + - run + + - name: Initialize Platform setup + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_setup + tags: + - initialize + - plat + - run + + - name: Set up Platform + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: setup + tags: + - plat + - run + + - name: Initialize Data Services setup + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_setup + tags: + - initialize + - run + + - name: Set up Data Services + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: setup + tags: + - run diff --git a/playbooks/pbc_teardown.yml b/playbooks/pbc_teardown.yml new file mode 100644 index 00000000..6d7f61e8 --- /dev/null +++ b/playbooks/pbc_teardown.yml @@ -0,0 +1,73 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Tear down CDP Public Cloud + hosts: "{{ target | default('localhost') }}" + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Validate Platform configuration + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: validate + tags: + - validate + - initialize + + - name: Validate Data Services configuration + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: validate + tags: + - validate + - initialize + + - name: Initialize Data Services teardown + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_teardown + tags: + - initialize + - run + - plat + + - name: Tear down Data Services + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: teardown + tags: + - run + - plat + + - name: Initialize Platform teardown + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_teardown + tags: + - initialize + - plat + + - name: Tear down Platform + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: teardown + tags: + - plat diff --git a/playbooks/pvc_base_postfix.yml b/playbooks/pvc_base_postfix.yml new file mode 100644 index 00000000..04ff74d2 --- /dev/null +++ b/playbooks/pvc_base_postfix.yml @@ -0,0 +1,166 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Fix Auto-TLS +- name: Auto-TLS Services Setup + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.cms_tls + when: autotls is defined and autotls == True + tags: + - autotls + - never +# ENDBLOCK # Fix Auto-TLS + +# STARTBLOCK # Setup HDFS Encryption +- name: Setup KTS HA + hosts: "{{ target | default('localhost') }}" + become: yes + gather_facts: no + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.deployment.services.kts_high_availability + public: yes + apply: + tags: + - kts + - full_cluster + when: + - "'kts_active' in groups" + - "'kts_passive' in groups" + tags: + - kts + - full_cluster + +- name: Handle KMS services + hosts: "{{ target | default('localhost') }}" + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.deployment.services.kms + when: "'kms_servers' in groups" + - role: cloudera.cluster.deployment.services.kms_ha + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Handle KMS services + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + become: no + roles: + - role: cloudera.cluster.operations.refresh_ranger_kms_repo + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Restart and re-deploy stale client configs + hosts: "{{ target | default('localhost') }}" + gather_facts: no + roles: + - role: cloudera.cluster.operations.restart_stale + when: "'kms_servers' in groups" + vars: + client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" + tags: + - kms + - restart_stale + - full_cluster + # ENDBLOCK # Setup HDFS Encryption + + # STARTBLOCK # WXM Setup +- name: Handle WXM Setup + hosts: "{{ target | default('cluster_master_nodes[0]') }}" + gather_facts: yes + tags: + - wxm + - full_cluster + tasks: + - name: Setup WXM + when: + - use_wxm | default(False) + - altus_key_id | length > 0 + - altus_private_key | length > 0 + import_role: + name: cloudera.cluster.deployment.services.wxm +# ENDBLOCK # WXM Setup + +- name: Post-Install for PvC on all cluster hosts + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + gather_facts: yes + tags: + - pvc + tasks: + - name: Add missing ExtJS for Oozie UI + include_role: + name: cloudera.cluster.config.services.oozie_ui + when: oozie_service_exists | default(false) + +- name: Post-Install Cloudera Manager and Cluster + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: yes + tags: + - pvc + tasks: + - name: Refresh CM Services Info + include_role: + name: cloudera.cluster.cloudera_manager.services_info + public: yes + vars: + cluster_name: "{{ cluster_name_base }}" + + - name: Fix Hue ticket lifetime for Free IPA + include_role: + name: cloudera.cluster.config.services.hue_ticket_lifetime + when: + - hue_service_exists | default(false) + - krb5_kdc_type == 'Red Hat IPA' + + - name: Set Cloudera Manager session timeout to 30d + include_role: + name: cloudera.cluster.cloudera_manager.session_timeout + +## TODO Believe this is version specific, in 7.1.7Spx solr plugin is NOT missing, needs work +# - name: Create missing Solr plugin for Ranger +# include_role: +# name: cloudera.cluster.config.services.solr_ranger_plugin +# when: +# - ranger_service_exists | default(false) +# - solr_service_exists | default(false) + + - name: Add Solr urls to Knox + include_role: + name: cloudera.cluster.config.services.solr_knox + when: knox_service_exists | default(false) and solr_service_exists | default(false) + + - name: Add missing TLS values for KMS + when: + - kms_service_exists | default(False) + - (autotls | default(False)) + include_role: + name: cloudera.cluster.config.services.kms_tls + +## TODO Believe this is version specific, in 7.1.7SPx Ranger policies are duplicated, needs work +# - name: Ranger extra policies to have enough rights with basic accounts +# when: ranger_service_exists | default(false) +# include_role: +# name: cloudera.cluster.config.services.ranger_pvc_default_policies + +# End run +### \ No newline at end of file diff --git a/playbooks/pvc_base_prereqs_ext.yml b/playbooks/pvc_base_prereqs_ext.yml new file mode 100644 index 00000000..0e4d67ad --- /dev/null +++ b/playbooks/pvc_base_prereqs_ext.yml @@ -0,0 +1,257 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Verify Inventory and Definition +- name: Verify inventory [verify_inventory] + hosts: "{{ target | default('localhost') }}" + gather_facts: no + roles: + - cloudera.cluster.verify.inventory + tags: + - verify + - verify_inventory + - default_cluster + - full_cluster + +- name: Verify definition [verify_definition] + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: yes + roles: + - cloudera.cluster.verify.definition + tags: + - verify + - verify_definition + - default_cluster + - full_cluster + +# Moved before parcel verification to allow rehosting +- name: Install custom parcel repository + hosts: "{{ target | default('custom_repo') }}" + become: yes + roles: + - cloudera.cluster.infrastructure.custom_repo + tags: + - custom_repo + - default_cluster + - full_cluster + +# Moved from verify_parcels to reduce duplication +- name: Verify definition [verify_parcels_and_roles] + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: yes + roles: + - cloudera.cluster.verify.parcels_and_roles + tags: + - verify + - verify_parcels + - default_cluster + - full_cluster +# ENDBLOCK # Verify Inventory and Definition + +# STARTBLOCK # Prepare Nodes +- name: Apply OS pre-requisite configurations + hosts: "{{ target | default('cloudera_manager, cluster, ca_server, ecs_nodes') }}" + become: yes + roles: + - cloudera.cluster.prereqs.os + tags: + - os + - default_cluster + - full_cluster + +- name: Apply OS Prereqs to ECS Nodes + hosts: "{{ target | default('ecs_nodes') }}" + gather_facts: yes + become: yes + tags: + - pvc + - os + - default_cluster + - full_cluster + tasks: + - name: Setup OS Prereqs for ECS Nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.pvc_ecs + +- name: Create local user accounts + hosts: "{{ target | default('cloudera_manager, cluster') }}" + become: yes + gather_facts: no + roles: + - cloudera.cluster.prereqs.user_accounts + tags: + - users + - default_cluster + - full_cluster + +- name: Create local users on ECS Nodes + hosts: "{{ target | default('ecs_nodes') }}" + become: yes + gather_facts: yes + tags: + - pvc + - users + - default_cluster + - full_cluster + tasks: + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + +- name: Install JDK + hosts: "{{ target | default('cloudera_manager, cluster, tls, krb5_server, ecs_nodes') }}" + become: yes + roles: + - cloudera.cluster.prereqs.jdk + tags: + - jdk + - security + - free_ipa + - kerberos + - tls + - default_cluster + - full_cluster + +# DB Connectors +- name: Install MySQL Connector + hosts: "{{ target | default('cloudera_manager, cluster') }}" + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.mysql_connector + when: database_type == 'mysql' or database_type == 'mariadb' + tags: + - mysql_connector + - full_cluster + +- name: Install Oracle Connector + hosts: "{{ target | default('cloudera_manager, cluster') }}" + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.oracle_connector + when: database_type == 'oracle' + tags: + - oracle_connector + - full_cluster +# ENDBLOCK # Prepare Nodes + +# STARTBLOCK # Create Cluster Service Infrastructure +- name: Install Kerberos Server + hosts: "{{ target | default('krb5_server') }}" + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_server + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Setup KRB5 clients + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_client + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Install CA server + hosts: "{{ target | default('ca_server') }}" + become: yes + roles: + - cloudera.cluster.infrastructure.ca_server + tags: + - security + - tls + - full_cluster + +- name: Install HAProxy + hosts: "{{ target | default('haproxy') }}" + become: yes + roles: + - cloudera.cluster.infrastructure.haproxy + tags: + - ha + - full_cluster + +# ENDBLOCK # Create Cluster Service Infrastructure +# STARTBLOCK # Prepare TLS + +- name: Build TLS keystores and truststores + hosts: "{{ target | default('tls') }}" + become: yes + gather_facts: no + roles: + - cloudera.cluster.security.tls_generate_csr + - cloudera.cluster.security.tls_signing + - cloudera.cluster.security.tls_install_certs + vars: + local_certs_dir: "{{ local_temp_dir }}/certs" + local_csrs_dir: "{{ local_temp_dir }}/csrs" + tags: + - tls + - full_cluster + +- name: Delete temp directories + hosts: "{{ target | default('localhost') }}" + become: no + gather_facts: no + tasks: + - file: + path: "{{ [local_temp_dir, dir] | path_join }}" + state: absent + loop: + - csrs + - certs + loop_control: + loop_var: dir + tags: + - always +# ENDBLOCK # Prepare TLS + +# STARTBLOCK # NiFi TLS +- name: Setup symlinks for NiFi TLS keystore and truststore + hosts: "{{ target | default('cluster') }}" + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.security.tls_nifi + when: > + (tls | default(False) + or manual_tls_cert_distribution | default(False)) + and not (autotls | default(False)) +# ENDBLOCK # NiFi TLS + +# STARTBLOCK # Install Cluster Service Infrastructure II +- name: Install RDBMS + hosts: "{{ target | default('db_server') }}" + become: yes + roles: + - cloudera.cluster.infrastructure.rdbms + tags: + - database + - default_cluster + - full_cluster +# ENDBLOCK # Install Cluster Service Infrastructure II diff --git a/playbooks/pvc_base_prereqs_int.yml b/playbooks/pvc_base_prereqs_int.yml new file mode 100644 index 00000000..a6e4c278 --- /dev/null +++ b/playbooks/pvc_base_prereqs_int.yml @@ -0,0 +1,207 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Install Cloudera Manager +- name: Install Cloudera Manager daemons + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.daemons + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager server + hosts: "{{ target | default('cloudera_manager') }}" + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.server + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager License + hosts: "{{ target | default('cloudera_manager') }}" + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.license + tags: + - cm + - license + - default_cluster + - full_cluster + +- name: Install Cloudera Manager agents + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.agent + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server for TLS + hosts: "{{ target | default('cloudera_manager') }}" + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.server_tls + when: tls | default(False) or manual_tls_cert_distribution | default(False) + tags: + - tls + - cm + - full_cluster + +- name: Configure Cloudera Manager agents + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.cloudera_manager.agent_config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager auth and accounts + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.external_auth + - role: cloudera.cluster.cloudera_manager.external_account + tags: + - cm + - full_cluster +# ENDBLOCK # Install Cloudera Manager + +# STARTBLOCK # Cloudera Manager Password +- name: Configure Cloudera Manager Password + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.admin_password.set + tags: + - cm + - default_cluster + - full_cluster + +- name: Check Cloudera Manager admin password + hosts: "{{ target | default('cloudera_manager, cluster') }}" + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.api_client + tags: + - cm + - default_cluster + - full_cluster +# ENDBLOCK # Cloudera Manager Password + +# STARTBLOCK # Prepare Security +- name: Enable Auto-TLS + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.autotls + when: autotls is defined and autotls == True + tags: + - autotls + - full_cluster + +- name: Install prerequisite packages for Kerberos + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + become: yes + roles: + - role: cloudera.cluster.prereqs.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - prereqs + - full_cluster + +- name: Configure Cloudera Manager server for Kerberos + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - full_cluster +# ENDBLOCK # Prepare Security + +# STARTBLOCK # Configure CM +- name: Restart Cloudera Manager Agents + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + gather_facts: no + become: yes + tasks: + - name: Restart Cloudera Manager Agents + meta: noop + notify: + - restart cloudera-scm-agent + tags: + - never + - restart_agents + +- name: Ensure that the agents are heartbeating + hosts: "{{ target | default('cloudera_manager, cluster, ecs_nodes') }}" + gather_facts: yes + any_errors_fatal: yes + roles: + - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat + when: cloudera_manager_agent_wait_for_heartbeat | default(True) + tags: + - heartbeat + - default_cluster + - full_cluster + +- name: Deploy Cloudera Management Service + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: no + roles: + - cloudera.cluster.deployment.services.mgmt + tags: + - mgmt + - default_cluster + - full_cluster + +- name: Preload parcels from custom repo to Cloudera Manager + hosts: "{{ target | default('cloudera_manager') }}" + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.preload_parcels + when: "'custom_repo' in groups" + tags: + - preload_parcels + - default_cluster + - full_cluster +# ENDBLOCK # Configure CM diff --git a/playbooks/pvc_base_setup.yml b/playbooks/pvc_base_setup.yml new file mode 100644 index 00000000..af646c2e --- /dev/null +++ b/playbooks/pvc_base_setup.yml @@ -0,0 +1,28 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Install Cluster +- name: Deploy clusters + hosts: "{{ target | default('cloudera_manager') }}" + gather_facts: yes + roles: + - cloudera.cluster.deployment.cluster + tags: + - cluster + - pvc + - default_cluster + - full_cluster +# ENDBLOCK # Install Cluster diff --git a/playbooks/pvc_base_teardown.yml b/playbooks/pvc_base_teardown.yml new file mode 100644 index 00000000..43ed18e9 --- /dev/null +++ b/playbooks/pvc_base_teardown.yml @@ -0,0 +1,68 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Teardown +# Teardown CA +- name: Teardown CA server + hosts: "{{ target | default('ca_server') }}" + environment: "{{ globals.env_vars }}" + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.infrastructure.ca_certs + tasks_from: clean.yml + tags: + - teardown_ca + +- name: Teardown security artifact directories + hosts: "{{ target | default('tls') }}" + environment: "{{ globals.env_vars }}" + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.security.tls_clean + when: "'tls' in groups" + tags: + - teardown_tls + +# Teardown Cluster +- name: Teardown ECS + hosts: "{{ target | default('ecs_nodes') }}" + environment: "{{ globals.env_vars }}" + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.teardown + tasks_from: teardown_ecs.yml + vars: + cluster: "{{ definition.clusters | selectattr('type', 'defined') | selectattr('type', 'search', 'ecs') | first }}" + tags: + - teardown_ecs + +- name: Cluster Teardown Process + hosts: "{{ target | default('all') }}" + environment: "{{ globals.env_vars }}" + gather_facts: no + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.teardown + tags: + - teardown_cluster + +# ENDBLOCK # Teardown \ No newline at end of file diff --git a/roles/init_deployment/defaults/main.yml b/roles/init_deployment/defaults/main.yml index 1661cbc1..f14b9568 100644 --- a/roles/init_deployment/defaults/main.yml +++ b/roles/init_deployment/defaults/main.yml @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Default definition path +definition_path: "./" + # Default Paths default_local_temp_dir: '/tmp' default_sshkey_path: '~/.ssh' @@ -27,21 +30,12 @@ default_ssh_key_suffix: _ssh_rsa default_cluster_definition_file: "vars/basic_cluster.yml" # Default Artefact Filenames -default_profile_path: "{{ [default_config_path, 'profiles'] | path_join }}" -default_profile_file: "default" default_definition_file: "definition.yml" default_cluster_file: "cluster.yml" -default_pre_setup_tasklist: "pre_setup.yml" -default_post_setup_tasklist: "post_setup.yml" -default_pre_teardown_tasklist: "pre_teardown.yml" -default_post_teardown_tasklist: "post_teardown.yml" include_inventory_file: '' auto_repo_mirror_file: "{{ [default_config_path, 'auto_repo_mirror.ini'] | path_join }}" -# Default behavior -use_default_cluster_definition: no - # Default Deployment Controls default_infra_deployment_engine: ansible default_infra_type: aws # azure, gcp diff --git a/roles/init_deployment/tasks/main.yml b/roles/init_deployment/tasks/main.yml index 2931a3ff..ef0ad06b 100644 --- a/roles/init_deployment/tasks/main.yml +++ b/roles/init_deployment/tasks/main.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,30 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Check versions -- name: Get Python packages - register: __python_env_packages - command: pip freeze - -- name: Get Ansible Collections - register: __ansible_collection_list - command: ansible-galaxy collection list - -- name: Get Ansible Roles - register: __ansible_role_list - command: ansible-galaxy role list - -- name: Log Runtime information - ansible.builtin.debug: - msg: - - "Ansible Version: {{ ansible_version | to_yaml }}" - - "Ansible Collections: {{ __ansible_collection_list.stdout | to_nice_yaml }}" - - "Ansible Roles: {{ __ansible_role_list.stdout }}" - - "Python Version: {{ ansible_python_version }}" - - "Python Packages: {{ __python_env_packages.stdout }}" - - "Runner Version: '{{ lookup('env', 'CLDR_BUILD_VER') }}'" - verbosity: 1 - - name: Marshall Deployment Definition ansible.builtin.include_tasks: marshall.yml diff --git a/roles/init_deployment/tasks/marshall.yml b/roles/init_deployment/tasks/marshall.yml index 3a457a3d..8b4ea3fe 100644 --- a/roles/init_deployment/tasks/marshall.yml +++ b/roles/init_deployment/tasks/marshall.yml @@ -40,25 +40,9 @@ # Set File Paths - name: Set Expected File Paths ansible.builtin.set_fact: - init__user_profile: "{{ abs_profile | default([profile_path | default(default_profile_path), profile | default(default_profile_file)] | path_join ) }}" init__definition_file: "{{ abs_definition | default( [definition_path, definition_file | default(default_definition_file)] | path_join ) }}" init__cluster_file: "{{ abs_cluster | default( [definition_path, cluster_file | default(default_cluster_file)] | path_join ) }}" init__auto_repo_mirror_artefact: "{{ auto_repo_mirror_file | default(auto_repo_mirror_file) }}" - init__pre_setup_tasklist: "{{ abs_pre_setup | default( [definition_path, pre_setup_tasklist | default(default_pre_setup_tasklist)] | path_join ) }}" - init__post_setup_tasklist: "{{ abs_post_setup | default( [definition_path, post_setup_tasklist | default(default_post_setup_tasklist)] | path_join ) }}" - init__pre_teardown_tasklist: "{{ abs_pre_teardown | default( [definition_path, pre_teardown_tasklist | default(default_pre_teardown_tasklist)] | path_join ) }}" - init__post_teardown_tasklist: "{{ abs_post_teardown | default( [definition_path, post_teardown_tasklist | default(default_post_teardown_tasklist)] | path_join ) }}" - -# Handle User Config -- name: Check for User Config file - register: __user_config_stat - ansible.builtin.stat: - path: "{{ init__user_profile }}" - -- name: Load User Config - when: __user_config_stat.stat.exists - ansible.builtin.include_vars: - file: "{{ __user_config_stat.stat.path }}" # Handle Definition File - name: Seek Definition files in Definition Path @@ -94,37 +78,18 @@ ansible.builtin.set_fact: init__cluster_definition_file: "{{ __clus_file_stat.stat.path }}" -# Override with default cluster definition if requested, regardless of files found -- name: Use default cluster definition as override if requested - when: use_default_cluster_definition | bool - block: - - name: Copy basic cluster definition to tmp - copy: - src: "{{ default_cluster_definition_file }}" - dest: /tmp/basic_cluster.yml - - - name: Set basic definition as target definition - ansible.builtin.set_fact: - init__cluster_definition_file: /tmp/basic_cluster.yml - -- name: Include vars from User Definition File to private dict to check for Globals - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - name: __def_vars - # Note that this depends on the earlier set_fact for globals to take precedence over include_vars - name: Include Cluster definition file for current localhost use after User Definition is Loaded ansible.builtin.include_vars: file: "{{ init__cluster_definition_file }}" - # Admin Password - name: Prompt User for a password if not provided in config or vault when: admin_password is undefined or admin_password | length < 2 block: - name: Prompt User for Password if not supplied no_log: true - pause: + ansible.builtin.pause: prompt: "No admin password found in profile.yml or extra_vars, or provided password too short; please provide a Password" register: __user_input_password @@ -208,8 +173,8 @@ AWS_PROFILE: "{{ globals.aws_profile | default(omit) }}" AWS_REGION: "{{ globals.region | default(omit) }}" -# This task is last to allow hard-coded 'globals' in the definition.yml to take top priority as a break-glass measure -- name: Merge overwrite globals from Definition file with Globals on User File - when: __def_vars.globals is defined - ansible.builtin.set_fact: - globals: "{{ globals | combine(__def_vars.globals, recursive=True) }}" \ No newline at end of file +# # This task is last to allow hard-coded 'globals' in the definition.yml to take top priority as a break-glass measure +# - name: Merge overwrite globals from Definition file with Globals on User File +# when: __def_vars.globals is defined +# ansible.builtin.set_fact: +# globals: "{{ globals | combine(__def_vars.globals, recursive=True) }}" \ No newline at end of file diff --git a/roles/init_deployment/tasks/runlevels.yml b/roles/init_deployment/tasks/runlevels.yml index 9d52aa91..91b4dda3 100644 --- a/roles/init_deployment/tasks/runlevels.yml +++ b/roles/init_deployment/tasks/runlevels.yml @@ -14,15 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Configure runlevel tags - ansible.builtin.set_fact: - run_teardown: "{{ 'teardown' in ansible_run_tags }}" - run_infrastructure: "{{ 'infra' in ansible_run_tags }}" - run_platform: "{{ 'plat' in ansible_run_tags }}" - run_pvc: "{{ 'pvc' in ansible_run_tags }}" - run_runtime: "{{ ansible_run_tags | difference(['infra', 'plat', 'teardown']) | length > 0 }}" +# - name: Configure runlevel tags +# ansible.builtin.set_fact: +# run_teardown: "{{ 'teardown' in ansible_run_tags }}" +# run_infrastructure: "{{ 'infra' in ansible_run_tags }}" +# run_platform: "{{ 'plat' in ansible_run_tags }}" +# run_pvc: "{{ 'pvc' in ansible_run_tags }}" +# run_runtime: "{{ ansible_run_tags | difference(['infra', 'plat', 'teardown']) | length > 0 }}" - name: Determine if Specific Roles should be called ansible.builtin.set_fact: - init__call_cdp_pvc: "{{ mgmt is defined or cluster is defined }}" + # init__call_cdp_pvc: "{{ mgmt is defined or cluster is defined }}" init__call_cdp_pbc: "{{ env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" \ No newline at end of file diff --git a/roles/init_deployment/tasks/validate.yml b/roles/init_deployment/tasks/validate.yml index 96b04546..09090341 100644 --- a/roles/init_deployment/tasks/validate.yml +++ b/roles/init_deployment/tasks/validate.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ fail_msg: "You must supply a valid Namespace" quiet: yes -- name: Check supplied Namespace +- name: Check supplied Namespace (AWS, GCP) when: - globals.infra_type != 'azure' - "'teardown' not in ansible_run_tags" diff --git a/roles/runtime/defaults/main.yml b/roles/runtime/defaults/main.yml index 62c0ee77..b9e0ffde 100644 --- a/roles/runtime/defaults/main.yml +++ b/roles/runtime/defaults/main.yml @@ -87,7 +87,7 @@ run__df_nodes_min: "{{ df.min_k8s_nodes | default(3) }}" run__df_nodes_max: "{{ df.max_k8s_nodes | default(5) }}" run__df_public_loadbalancer: "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}" run__df_lb_ip_ranges: "{{ df.loadbalancer_ip_ranges | default([]) }}" -run__df_k8s_ip_ranges: "{{ df.k8s_ip_ranges | default([]) }}" +run__df_k8s_ip_ranges: "{{ df.k8s_ip_ranges | default([]) }}" run__df_cluster_subnets: "{{ df.cluster_subnets | default(omit) }}" run__df_cluster_subnets_filter: "{{ df.cluster_subnets_filter | default(omit) }}" run__df_lb_subnets: "{{ df.loadbalancer_subnets | default(omit) }}" diff --git a/roles/runtime/tasks/setup_base.yml b/roles/runtime/tasks/setup_base.yml index 7b0f2e6a..6baac52f 100644 --- a/roles/runtime/tasks/setup_base.yml +++ b/roles/runtime/tasks/setup_base.yml @@ -85,6 +85,7 @@ skip_validation: "{{ __de_config_item.raw.skip_validation | default(omit) }}" tags: "{{ __de_config_item.raw.tags | default(omit) }}" use_ssd: "{{ __de_config_item.raw.use_ssd | default(omit) }}" + loadbalancer_ips: "{{ __de_config_item.raw.loadbalancer_ips | default(omit) }}" whitelist_ips: "{{ __de_config_item.raw.whitelist_ips | default(omit) }}" loop_control: loop_var: __de_config_item