From 21100947d8e2ba7f21ee021201bcf2b0d1f1a331 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 30 Jun 2022 19:08:10 -0400 Subject: [PATCH 1/5] Initial version for CDP Public Cloud setup and teardown using definition-based infrastructure Signed-off-by: Webster Mudge --- LICENSE | 2 +- cloud.yml | 98 ------------ cloud_setup.yml | 73 +++++++++ cloud_teardown.yml | 50 +++++++ cluster.yml => cluster_setup.yml | 50 +------ cluster_teardown.yml | 53 +++++++ default/post_setup.yml | 5 + default/post_teardown.yml | 5 + default/pre_setup.yml | 5 + default/pre_teardown.yml | 5 + examples/refactor/definition.yml | 0 examples/refactor/post_setup.yml | 5 + examples/refactor/post_teardown.yml | 5 + examples/refactor/pre_setup.yml | 56 +++++++ examples/refactor/pre_teardown.yml | 18 +++ main.yml | 115 +++++++------- roles/download_mirror/NOTES.md | 35 +++++ .../tasks/inject_download_mirror.yml | 0 .../tasks/main.yml} | 9 ++ .../tasks/populate_download_mirror.yml | 0 .../tasks/prepare_download_mirror.yml | 19 +++ .../tasks/update_download_mirror_cache.yml | 31 ++++ roles/dynamic_inventory/NOTES.md | 45 ++++++ roles/dynamic_inventory/defaults/main.yml | 5 + .../tasks/clean_dynamic_inventory.yml | 0 roles/dynamic_inventory/tasks/main.yml | 61 ++++++++ .../tasks/persist_dynamic_inventory.yml | 0 .../tasks/refresh_inventory.yml | 2 + roles/initialize_parameters/NOTES.md | 15 ++ .../defaults/main.yml | 2 +- .../tasks/distribute_facts_to_inventory.yml | 0 .../tasks/main.yml} | 141 ++---------------- .../vars}/basic_cluster.yml | 0 roles/runlevels/tasks/main.yml | 43 ++++++ roles/terraform_state/NOTES.MD | 16 ++ .../tasks/auto_terraform_state.yml | 0 roles/terraform_state/tasks/main.yml | 36 +++++ 37 files changed, 675 insertions(+), 330 deletions(-) delete mode 100644 cloud.yml create mode 100644 cloud_setup.yml create mode 100644 cloud_teardown.yml rename cluster.yml => cluster_setup.yml (93%) create mode 100644 cluster_teardown.yml create mode 100644 default/post_setup.yml create mode 100644 default/post_teardown.yml create mode 100644 default/pre_setup.yml create mode 100644 default/pre_teardown.yml create mode 100644 examples/refactor/definition.yml create mode 100644 examples/refactor/post_setup.yml create mode 100644 examples/refactor/post_teardown.yml create mode 100644 examples/refactor/pre_setup.yml create mode 100644 examples/refactor/pre_teardown.yml create mode 100644 roles/download_mirror/NOTES.md rename roles/{cloudera_deploy => download_mirror}/tasks/inject_download_mirror.yml (100%) rename roles/{cloudera_deploy/tasks/prepare_download_mirror.yml => download_mirror/tasks/main.yml} (94%) rename roles/{cloudera_deploy => download_mirror}/tasks/populate_download_mirror.yml (100%) create mode 100644 roles/download_mirror/tasks/prepare_download_mirror.yml create mode 100644 roles/download_mirror/tasks/update_download_mirror_cache.yml create mode 100644 roles/dynamic_inventory/NOTES.md create mode 100644 roles/dynamic_inventory/defaults/main.yml rename roles/{cloudera_deploy => dynamic_inventory}/tasks/clean_dynamic_inventory.yml (100%) create mode 100644 roles/dynamic_inventory/tasks/main.yml rename roles/{cloudera_deploy => dynamic_inventory}/tasks/persist_dynamic_inventory.yml (100%) rename roles/{cloudera_deploy => dynamic_inventory}/tasks/refresh_inventory.yml (97%) create mode 100644 roles/initialize_parameters/NOTES.md rename roles/{cloudera_deploy => initialize_parameters}/defaults/main.yml (96%) rename roles/{cloudera_deploy => initialize_parameters}/tasks/distribute_facts_to_inventory.yml (100%) rename roles/{cloudera_deploy/tasks/init.yml => initialize_parameters/tasks/main.yml} (72%) rename roles/{cloudera_deploy/defaults => initialize_parameters/vars}/basic_cluster.yml (100%) create mode 100644 roles/runlevels/tasks/main.yml create mode 100644 roles/terraform_state/NOTES.MD rename roles/{cloudera_deploy => terraform_state}/tasks/auto_terraform_state.yml (100%) create mode 100644 roles/terraform_state/tasks/main.yml diff --git a/LICENSE b/LICENSE index 0324e14..7770734 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2021 Cloudera, Inc. +Copyright 2022 Cloudera, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cloud.yml b/cloud.yml deleted file mode 100644 index 43d48cd..0000000 --- a/cloud.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This play is imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml -- name: Marshal Cloud Deployment - hosts: localhost - environment: "{{ globals.env_vars }}" - gather_facts: yes - tasks: - - name: Import the Sequence execution Role for Cloud Run - ansible.builtin.import_role: - name: cloudera.exe.sequence - when: init__call_cloud_role | bool - - - name: Persist Dynamic Inventory to Definition Path if exists - tags: always - when: - - infra__dynamic_inventory_host_entries is defined - - infra__dynamic_inventory_host_entries | length > 0 - - init__dynamic_inventory_template is defined - - init__dynamic_inventory_template | length > 0 - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: persist_dynamic_inventory - apply: - tags: always - -# TODO: Set timeout waiting for utility VM to be ready -- name: Process Download Mirror on Utility VM - hosts: cldr_utility - tags: always - gather_facts: yes - tasks: - - name: Fetch necessary variables from Ansible Controller - ansible.builtin.set_fact: - globals: "{{ hostvars['localhost']['globals'] }}" - init__download_mirror_artefact: "{{ hostvars['localhost']['init__download_mirror_artefact'] }}" - - - name: Prepare Cloudera Subscription Credentials - ansible.builtin.include_role: - name: cloudera.cluster.deployment.credential - when: globals.cloudera_license_file is defined - vars: - cloudera_manager_license_file: "{{ globals.cloudera_license_file }}" - - - name: Populate the Download Mirror with new files - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: populate_download_mirror - when: globals.download_mirror_targets | length > 0 - -- name: Update relevant Download Mirror Cache listing - hosts: localhost - tags: always - gather_facts: no - tasks: - - name: Refresh Listing of target cache contents - when: - - init__download_mirror_bucket_name is defined - - "'teardown' not in ansible_run_tags" - register: __infra_download_mirror_listing - failed_when: - - __download_mirror_lookup_initial.s3_keys is not defined - - "'cannot be found' not in __download_mirror_lookup_initial.msg" - amazon.aws.aws_s3: - bucket: "{{ init__download_mirror_bucket_name }}" - mode: list - - - name: Prepare updated Download Mirror contents as URLs - when: __infra_download_mirror_listing.s3_keys is defined - loop: "{{ __infra_download_mirror_listing.s3_keys }}" - loop_control: - loop_var: __download_mirror_s3_urls_item - ansible.builtin.set_fact: - __download_mirror_url_listing: "{{ __download_mirror_url_listing | default([]) + [['https:/', init__download_mirror_bucket_name + '.s3.amazonaws.com', __download_mirror_s3_urls_item ] | join('/') ] }}" - - - name: Persist Download Mirror to Definition path - when: - - __download_mirror_url_listing is defined - - __download_mirror_url_listing | length > 0 - community.general.ini_file: - path: "{{ init__download_mirror_artefact }}" - section: "{{ globals.infra_type }}:{{ globals.region }}" - option: "{{ init__download_mirror_bucket_name }}" - value: "{{ __download_mirror_url_listing }}" \ No newline at end of file diff --git a/cloud_setup.yml b/cloud_setup.yml new file mode 100644 index 0000000..39eab83 --- /dev/null +++ b/cloud_setup.yml @@ -0,0 +1,73 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up CDP Public Cloud + hosts: localhost + connection: local + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# Inherits 'plat' and 'run' from playbook import +# Applies 'validate', and 'initialize' via the role imports +# By using imports vs includes for roles, we can propagate +# tags via --list-tags + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Validate Platform configuration + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: validate + tags: + - validate + + - name: Validate Data Services configuration + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: validate + tags: + - validate + + - name: Initialize Platform setup + #when: sequence__setup_plat | bool + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_setup + tags: + - initialize + + - name: Set up Platform + #when: sequence__setup_plat | bool + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: setup + + - name: Initialize Data Services setup + #when: sequence__setup_runtime | bool + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_setup + tags: + - initialize + + - name: Set up Data Services + #when: sequence__setup_runtime | bool + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: setup diff --git a/cloud_teardown.yml b/cloud_teardown.yml new file mode 100644 index 0000000..2f63635 --- /dev/null +++ b/cloud_teardown.yml @@ -0,0 +1,50 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Tear down CDP Public Cloud + hosts: localhost + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + +# Inherits 'infra', 'teardown', and 'never' from the playbook import +# Does not apply other tags via the role imports +# By using imports vs includes for roles, we can propagate +# tags via --list-tags + +# TODO Need to propagate 'ml', 'dw', etc. to selected tasks within the roles, +# including those that would otherwise be 'always' - in this context; 'always' +# should be reserved for the initialization of cloudera-deploy + + - name: Initialize Data Services teardown + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: initialize_teardown + + - name: Tear down Data Services + ansible.builtin.import_role: + name: cloudera.exe.runtime + tasks_from: teardown + + - name: Initialize Platform teardown + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: initialize_teardown + + - name: Tear down Platform + ansible.builtin.import_role: + name: cloudera.exe.platform + tasks_from: teardown diff --git a/cluster.yml b/cluster_setup.yml similarity index 93% rename from cluster.yml rename to cluster_setup.yml index eedf181..e2d69c7 100644 --- a/cluster.yml +++ b/cluster_setup.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2022 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ # STARTBLOCK # Init run - name: Init Cluster Deployment Tasks hosts: localhost - tags: always gather_facts: no tasks: # TODO: Move to dependent roles with delegation if necessary @@ -34,10 +33,11 @@ state: directory loop_control: loop_var: __dir + tags: + - always - name: Check Inventory Connectivity hosts: all - tags: always gather_facts: no tasks: - name: Check connectivity to Inventory @@ -45,6 +45,8 @@ - name: Gather facts from connected inventory setup: + tags: + - always # ENDBLOCK # Init run # STARTBLOCK # Verify Inventory and Definition @@ -529,48 +531,6 @@ - full_cluster # ENDBLOCK # Setup HDFS Encryption -# STARTBLOCK # Teardown -# Teardown CA - -- name: Teardown CA server - hosts: ca_server - gather_facts: no - become: yes - tasks: - - ansible.builtin.include_role: - name: cloudera.cluster.infrastructure.ca_certs - tasks_from: clean.yml - tags: - - teardown_ca - - teardown_all - - never - -- name: Teardown security artifact directories - hosts: tls - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.security.tls_clean - when: "'tls' in groups" - tags: - - teardown_tls - - teardown_all - - never - -# Teardown Cluster - -- name: Teardown - hosts: all - gather_facts: no - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.teardown - tags: - - teardown_cluster - - teardown_all - - never -# ENDBLOCK # Teardown # End run ### diff --git a/cluster_teardown.yml b/cluster_teardown.yml new file mode 100644 index 0000000..4528c8c --- /dev/null +++ b/cluster_teardown.yml @@ -0,0 +1,53 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STARTBLOCK # Teardown +# Teardown CA + +- name: Teardown CA server + hosts: ca_server + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.infrastructure.ca_certs + tasks_from: clean.yml + tags: + - teardown_ca + +- name: Teardown security artifact directories + hosts: tls + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.security.tls_clean + when: "'tls' in groups" + tags: + - teardown_tls + +# Teardown Cluster + +- name: Teardown + hosts: all + gather_facts: no + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.teardown + tags: + - teardown_cluster + +# ENDBLOCK # Teardown \ No newline at end of file diff --git a/default/post_setup.yml b/default/post_setup.yml new file mode 100644 index 0000000..3edd289 --- /dev/null +++ b/default/post_setup.yml @@ -0,0 +1,5 @@ +--- + +- name: Default post-deploy set up + hosts: localhost + gather_facts: no diff --git a/default/post_teardown.yml b/default/post_teardown.yml new file mode 100644 index 0000000..7fee23e --- /dev/null +++ b/default/post_teardown.yml @@ -0,0 +1,5 @@ +--- + +- name: Default post-deploy tear down + hosts: localhost + gather_facts: no diff --git a/default/pre_setup.yml b/default/pre_setup.yml new file mode 100644 index 0000000..d3d3a84 --- /dev/null +++ b/default/pre_setup.yml @@ -0,0 +1,5 @@ +--- + +- name: Default pre-deploy set up + hosts: localhost + gather_facts: no diff --git a/default/pre_teardown.yml b/default/pre_teardown.yml new file mode 100644 index 0000000..4801ba4 --- /dev/null +++ b/default/pre_teardown.yml @@ -0,0 +1,5 @@ +--- + +- name: Default pre-deploy tear down + hosts: localhost + gather_facts: no diff --git a/examples/refactor/definition.yml b/examples/refactor/definition.yml new file mode 100644 index 0000000..e69de29 diff --git a/examples/refactor/post_setup.yml b/examples/refactor/post_setup.yml new file mode 100644 index 0000000..3edd289 --- /dev/null +++ b/examples/refactor/post_setup.yml @@ -0,0 +1,5 @@ +--- + +- name: Default post-deploy set up + hosts: localhost + gather_facts: no diff --git a/examples/refactor/post_teardown.yml b/examples/refactor/post_teardown.yml new file mode 100644 index 0000000..7fee23e --- /dev/null +++ b/examples/refactor/post_teardown.yml @@ -0,0 +1,5 @@ +--- + +- name: Default post-deploy tear down + hosts: localhost + gather_facts: no diff --git a/examples/refactor/pre_setup.yml b/examples/refactor/pre_setup.yml new file mode 100644 index 0000000..4614607 --- /dev/null +++ b/examples/refactor/pre_setup.yml @@ -0,0 +1,56 @@ +--- + +- name: Example pre-deploy set up playbook + hosts: localhost + connection: local + gather_facts: no + tasks: + +# Inherits 'infra', 'plat', and 'run' from playbook import +# Applies 'validate' and 'initialize' via role import +# Data Service runtimes, like 'dw' and 'ml', can applied selectively +# within a role's tasks or via role import, as below + + - name: Start pre-deployment tasks + debug: + msg: This is the start. + + - name: Validate Infrastructure configuration + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: validate + tags: + - validate + - de + - df + - dh + - dw + - ml + - opdb + + - name: Initialize Infrastructure setup + #when: sequence__setup_infra | bool + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: initialize_setup + tags: + - initialize + - de + - df + - dh + - dw + - ml + - opdb + + - name: Set up Infrastructure + #when: sequence__setup_infra | bool + ansible.builtin.include_role: + name: cloudera.exe.infrastructure + tasks_from: setup + tags: + - de + - df + - dh + - dw + - ml + - opdb \ No newline at end of file diff --git a/examples/refactor/pre_teardown.yml b/examples/refactor/pre_teardown.yml new file mode 100644 index 0000000..5c7f698 --- /dev/null +++ b/examples/refactor/pre_teardown.yml @@ -0,0 +1,18 @@ +--- + +- name: Example pre-deploy tear down + hosts: localhost + gather_facts: no + tasks: + + # Inherits 'teardown' and 'never' from playbook import + + - name: Initialize Infrastructure teardown + ansible.builtin.import_role: + name: cloudera.exe.infrastructure + tasks_from: initialize_teardown + + - name: Tear down Infrastructure + ansible.builtin.include_role: + name: cloudera.exe.infrastructure + tasks_from: teardown diff --git a/main.yml b/main.yml index 323aa79..59b3c0f 100644 --- a/main.yml +++ b/main.yml @@ -1,6 +1,6 @@ --- -# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# Copyright 2022 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,72 +14,75 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Primary entrypoint for all CDP Installations +# Primary entrypoint for all CDP installations (Public Cloud, Private Cloud) +# +# There are five "core" runlevels: +# - validate +# - initialize +# - infra +# - plat +# - run +# - teardown +# +# In this context, 'run' means 'runtime' or 'application'. The platform level +# is now both CDP Public Cloud SDX and Data Services, which is a change from +# from before where 'plat' referred to the DL and SDX and 'run' referred to the +# Data Services. +# +# Individual Data Services (and Datahubs) should be tagged _within_ the +# 'cloudera.exe.*' roles themselves: +# - de +# - df +# - dh +# - dw +# - ml +# - opdb -- name: Init Cloudera-Deploy Run +- name: Establish Cloudera Deploy runlevels hosts: localhost + gather_facts: no + run_once: True tags: always - gather_facts: yes tasks: - - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: init + - ansible.builtin.import_role: + name: runlevels -- name: Import the Cloud Interactions Playbook - import_playbook: cloud.yml - -- name: Teardown Cleanup +- name: Initialize Cloudera Deploy parameters hosts: localhost - tags: [teardown,never] gather_facts: yes + tags: always tasks: - - name: Remove current Dynamic Inventory file from Definition Path if exists - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: clean_dynamic_inventory + - ansible.builtin.import_role: + name: initialize_parameters # Should/could be more granular - - name: Remove Terraform remote state resources if requested - when: - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: auto_terraform_state +- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'post_teardown.yml'] | path_join }}" + tags: ['never', 'plat', 'infra', 'teardown'] + when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_teardown -- name: Prepare for Cloudera Cluster Run - hosts: localhost - tags: always - gather_facts: yes - tasks: - - name: Load Static Inventory file if present - when: mgmt is defined | bool - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: refresh_inventory - vars: - include_inventory_file: "{{ init__dynamic_inventory_artefact }}" +- ansible.builtin.import_playbook: cloud_teardown.yml + tags: ['never', 'infra', 'teardown'] + when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + +- ansible.builtin.import_playbook: cluster_teardown.yml + tags: ['never', 'infra', 'teardown'] + when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + +- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'pre_setup.yml'] | path_join }}" + tags: ['run', 'plat', 'infra'] + when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - name: Inject Download Mirror if requested - when: - - "'teardown' not in ansible_run_tags" - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: inject_download_mirror +- ansible.builtin.import_playbook: cloud_setup.yml + tags: ['run', 'plat'] + when: hostvars.localhost.run_platform or hostvars.localhost.run_runtime - - name: Distribute Facts to Inventory Hosts - when: - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: distribute_facts_to_inventory +- ansible.builtin.import_playbook: cluster_setup.yml + tags: ['run', 'plat'] + when: hostvars.localhost.run_platform or hostvars.localhost.run_runtime -- name: Import the Cluster Interactions Playbook - when: - - "'cluster' in groups" - - mgmt is defined | bool - import_playbook: cluster.yml +- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'post_setup.yml'] | path_join }}" + tags: ['run'] + when: hostvars.localhost.run_runtime -- name: Execute the Application Playbook - import_playbook: "{{ [definition_path, 'application.yml'] | path_join }}" +- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'pre_teardown.yml'] | path_join }}" + tags: ['never', 'teardown'] + when: hostvars.localhost.run_teardown diff --git a/roles/download_mirror/NOTES.md b/roles/download_mirror/NOTES.md new file mode 100644 index 0000000..7f7f38e --- /dev/null +++ b/roles/download_mirror/NOTES.md @@ -0,0 +1,35 @@ +# USAGE + +```yml +- name: Prepare for Cloudera Cluster Run + hosts: localhost + tags: always + gather_facts: yes + tasks: + - name: Inject Download Mirror if requested + when: + - "'teardown' not in ansible_run_tags" + - "'cluster' in groups" + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: inject_download_mirror + +# TODO: Set timeout waiting for utility VM to be ready +- name: Process Download Mirror on Utility VM + hosts: cldr_utility + tags: always + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: download_mirror + tasks_from: prepare_download_mirror.yml + +- name: Update relevant Download Mirror Cache listing + hosts: localhost + tags: always + gather_facts: no + tasks: + - ansible.builtin.include_role: + name: download_mirror + tasks_from: update_download_mirror_cache.yml +``` \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/inject_download_mirror.yml b/roles/download_mirror/tasks/inject_download_mirror.yml similarity index 100% rename from roles/cloudera_deploy/tasks/inject_download_mirror.yml rename to roles/download_mirror/tasks/inject_download_mirror.yml diff --git a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml b/roles/download_mirror/tasks/main.yml similarity index 94% rename from roles/cloudera_deploy/tasks/prepare_download_mirror.yml rename to roles/download_mirror/tasks/main.yml index 10ef568..c5c0e88 100644 --- a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml +++ b/roles/download_mirror/tasks/main.yml @@ -1,5 +1,14 @@ --- +# Download Mirror +#- name: Prepare for Download Mirror Population if requested and necessary +# when: +# - use_download_mirror | default(default_enable_download_mirror) | bool +# - "'teardown' not in ansible_run_tags" +# ansible.builtin.include_role: +# name: cloudera_deploy +# tasks_from: prepare_download_mirror + # Copyright 2021 Cloudera, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/roles/cloudera_deploy/tasks/populate_download_mirror.yml b/roles/download_mirror/tasks/populate_download_mirror.yml similarity index 100% rename from roles/cloudera_deploy/tasks/populate_download_mirror.yml rename to roles/download_mirror/tasks/populate_download_mirror.yml diff --git a/roles/download_mirror/tasks/prepare_download_mirror.yml b/roles/download_mirror/tasks/prepare_download_mirror.yml new file mode 100644 index 0000000..5fce0ee --- /dev/null +++ b/roles/download_mirror/tasks/prepare_download_mirror.yml @@ -0,0 +1,19 @@ +--- + +- name: Fetch necessary variables from Ansible Controller + ansible.builtin.set_fact: + globals: "{{ hostvars['localhost']['globals'] }}" + init__download_mirror_artefact: "{{ hostvars['localhost']['init__download_mirror_artefact'] }}" + +- name: Prepare Cloudera Subscription Credentials + ansible.builtin.include_role: + name: cloudera.cluster.deployment.credential + when: globals.cloudera_license_file is defined + vars: + cloudera_manager_license_file: "{{ globals.cloudera_license_file }}" + +- name: Populate the Download Mirror with new files + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: populate_download_mirror + when: globals.download_mirror_targets | length > 0 diff --git a/roles/download_mirror/tasks/update_download_mirror_cache.yml b/roles/download_mirror/tasks/update_download_mirror_cache.yml new file mode 100644 index 0000000..6cd77c4 --- /dev/null +++ b/roles/download_mirror/tasks/update_download_mirror_cache.yml @@ -0,0 +1,31 @@ +--- + +- name: Refresh Listing of target cache contents + when: + - init__download_mirror_bucket_name is defined + - "'teardown' not in ansible_run_tags" + register: __infra_download_mirror_listing + failed_when: + - __download_mirror_lookup_initial.s3_keys is not defined + - "'cannot be found' not in __download_mirror_lookup_initial.msg" + amazon.aws.aws_s3: + bucket: "{{ init__download_mirror_bucket_name }}" + mode: list + +- name: Prepare updated Download Mirror contents as URLs + when: __infra_download_mirror_listing.s3_keys is defined + loop: "{{ __infra_download_mirror_listing.s3_keys }}" + loop_control: + loop_var: __download_mirror_s3_urls_item + ansible.builtin.set_fact: + __download_mirror_url_listing: "{{ __download_mirror_url_listing | default([]) + [['https:/', init__download_mirror_bucket_name + '.s3.amazonaws.com', __download_mirror_s3_urls_item ] | join('/') ] }}" + +- name: Persist Download Mirror to Definition path + when: + - __download_mirror_url_listing is defined + - __download_mirror_url_listing | length > 0 + community.general.ini_file: + path: "{{ init__download_mirror_artefact }}" + section: "{{ globals.infra_type }}:{{ globals.region }}" + option: "{{ init__download_mirror_bucket_name }}" + value: "{{ __download_mirror_url_listing }}" \ No newline at end of file diff --git a/roles/dynamic_inventory/NOTES.md b/roles/dynamic_inventory/NOTES.md new file mode 100644 index 0000000..dde0e52 --- /dev/null +++ b/roles/dynamic_inventory/NOTES.md @@ -0,0 +1,45 @@ +# USAGE + +```yml +- name: Prepare for Cloudera Cluster Run + hosts: localhost + tags: always + gather_facts: yes + tasks: + - name: Load Static Inventory file if present + when: mgmt is defined | bool + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: refresh_inventory + vars: + include_inventory_file: "{{ init__dynamic_inventory_artefact }}" + +- name: Marshal Cloud Deployment + hosts: localhost + environment: "{{ globals.env_vars }}" + gather_facts: yes + tasks: + - name: Persist Dynamic Inventory to Definition Path if exists + tags: always + when: + - infra__dynamic_inventory_host_entries is defined + - infra__dynamic_inventory_host_entries | length > 0 + - init__dynamic_inventory_template is defined + - init__dynamic_inventory_template | length > 0 + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: persist_dynamic_inventory + apply: + tags: always +``` + +```yml +- name: Teardown Cleanup + hosts: localhost + gather_facts: yes + tasks: + - name: Remove current Dynamic Inventory file from Definition Path if exists + ansible.builtin.include_role: + name: dynamic_inventory + tasks_from: clean_dynamic_inventory +``` \ No newline at end of file diff --git a/roles/dynamic_inventory/defaults/main.yml b/roles/dynamic_inventory/defaults/main.yml new file mode 100644 index 0000000..e0a8263 --- /dev/null +++ b/roles/dynamic_inventory/defaults/main.yml @@ -0,0 +1,5 @@ +--- + +init__dynamic_inventory_template: "{{ abs_template | default( [definition_path, inventory_template | default(default_inventory_template)] | path_join ) }}" +init__dynamic_inventory_artefact: "{{ abs_inventory | default( [definition_path, static_inventory | default(default_static_inventory) ] | path_join ) }}" +init__download_mirror_artefact: "{{ download_mirror_file | default(download_mirror_file) }}" diff --git a/roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml b/roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml similarity index 100% rename from roles/cloudera_deploy/tasks/clean_dynamic_inventory.yml rename to roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml diff --git a/roles/dynamic_inventory/tasks/main.yml b/roles/dynamic_inventory/tasks/main.yml new file mode 100644 index 0000000..a5deff4 --- /dev/null +++ b/roles/dynamic_inventory/tasks/main.yml @@ -0,0 +1,61 @@ +# Parcel Distro +- name: Determine preferred Parcel Distribution + ansible.builtin.set_fact: + init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}" + +# Read in Dynamic Inventory +- name: Seek Inventory Template in Definition Path + register: __di_template_stat + ansible.builtin.stat: + path: "{{ init__dynamic_inventory_template }}" + +# inventory_dir is not defined when a user passes in an inventory with -i, so it is a useful check +# No point loading Dynamic Inventory if we are not doing infrastructure in this run +- name: Handle Dynamic Inventory Template + when: + - inventory_dir is defined + - __di_template_stat.stat.exists + block: + - name: Load in Dynamic Inventory Template + include_tasks: refresh_inventory.yml + vars: + include_inventory_file: "{{ __di_template_stat.stat.path }}" + + - name: Print Dynamic Inventory groups to debug at Verbosity 3 + debug: + msg: "{{ groups }}" + verbosity: 3 + + - name: Check expected minimum host groups appear in Inventory + ansible.builtin.assert: + quiet: yes + that: + - groups.cluster is defined + - groups.cloudera_manager is defined + fail_msg: "Parsed Inventory Template did not contain minimum expected groups for a Cloudera Cluster deployment" + + - name: Check a custom_repo is part of the cluster definition if using Download Mirror + when: use_download_mirror | default(default_enable_download_mirror) | bool + ansible.builtin.assert: + that: "'custom_repo' in groups" + fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" + + - name: Extract list of hosts from Dynamic Inventory Template + when: groups.all | length > 0 + ansible.builtin.set_fact: + __dynamic_inventory_host_list: "{{ groups.all | difference(['localhost']) }}" + + - name: Set Dynamic Inventory host count in Globals + when: __dynamic_inventory_host_list | length > 0 + ansible.builtin.set_fact: + globals: "{{ globals | default({}) | combine( __di_entry | default(omit), recursive=True ) }}" + loop_control: + loop_var: __di_entry + loop: + - dynamic_inventory: + vm: + count: "{{ __dynamic_inventory_host_list | count }}" + os: "{{ init__parcel_distro }}" + always: + - name: Remove Dynamic Inventory Template from current inventory + include_tasks: refresh_inventory.yml \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml b/roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml similarity index 100% rename from roles/cloudera_deploy/tasks/persist_dynamic_inventory.yml rename to roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml diff --git a/roles/cloudera_deploy/tasks/refresh_inventory.yml b/roles/dynamic_inventory/tasks/refresh_inventory.yml similarity index 97% rename from roles/cloudera_deploy/tasks/refresh_inventory.yml rename to roles/dynamic_inventory/tasks/refresh_inventory.yml index 5313d2e..ce163fc 100644 --- a/roles/cloudera_deploy/tasks/refresh_inventory.yml +++ b/roles/dynamic_inventory/tasks/refresh_inventory.yml @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO Consider converting to a handler? + - name: Check for additional inventory file ansible.builtin.stat: path: "{{ include_inventory_file }}" diff --git a/roles/initialize_parameters/NOTES.md b/roles/initialize_parameters/NOTES.md new file mode 100644 index 0000000..a9d9e61 --- /dev/null +++ b/roles/initialize_parameters/NOTES.md @@ -0,0 +1,15 @@ +# USAGE + +```yml +- name: Prepare for Cloudera Cluster Run + hosts: localhost + tags: always + gather_facts: yes + tasks: + - name: Distribute Facts to Inventory Hosts + when: + - "'cluster' in groups" + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: distribute_facts_to_inventory +``` \ No newline at end of file diff --git a/roles/cloudera_deploy/defaults/main.yml b/roles/initialize_parameters/defaults/main.yml similarity index 96% rename from roles/cloudera_deploy/defaults/main.yml rename to roles/initialize_parameters/defaults/main.yml index bcdebbe..9715d25 100644 --- a/roles/cloudera_deploy/defaults/main.yml +++ b/roles/initialize_parameters/defaults/main.yml @@ -25,7 +25,7 @@ default_ssh_key_suffix: _ssh_rsa default_download_mirror_prefix: cache # Default Basic Cluster Config -default_cluster_definition_file: "defaults/basic_cluster.yml" +default_cluster_definition_file: "basic_cluster.yml" # Default Artefact Filenames default_profile_path: "{{ [default_config_path, 'profiles'] | path_join }}" diff --git a/roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml b/roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml similarity index 100% rename from roles/cloudera_deploy/tasks/distribute_facts_to_inventory.yml rename to roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml diff --git a/roles/cloudera_deploy/tasks/init.yml b/roles/initialize_parameters/tasks/main.yml similarity index 72% rename from roles/cloudera_deploy/tasks/init.yml rename to roles/initialize_parameters/tasks/main.yml index 27eb1ca..2ac9f98 100644 --- a/roles/cloudera_deploy/tasks/init.yml +++ b/roles/initialize_parameters/tasks/main.yml @@ -29,16 +29,14 @@ - name: Log Runtime information ansible.builtin.debug: - msg: "{{ __runtime_item }}" - loop_control: - loop_var: __runtime_item - loop: - - "Ansible Version: {{ ansible_version }}" - - "Ansible Collections: {{ __ansible_collection_list.stdout }}" - - "Ansible Roles: {{ __ansible_role_list.stdout }}" - - "Python Version: {{ ansible_python_version }}" - - "Python Packages: {{ __python_env_packages.stdout }}" - - "Runner Version: '{{ lookup('env', 'CLDR_BUILD_VER') }}'" + msg: + - "Ansible Version: {{ ansible_version | to_yaml }}" + - "Ansible Collections: {{ __ansible_collection_list.stdout | to_nice_yaml }}" + - "Ansible Roles: {{ __ansible_role_list.stdout }}" + - "Python Version: {{ ansible_python_version }}" + - "Python Packages: {{ __python_env_packages.stdout }}" + - "Runner Version: '{{ lookup('env', 'CLDR_BUILD_VER') }}'" + verbosity: 1 # Check path - name: Check a Definition path has been supplied @@ -68,9 +66,6 @@ init__user_profile: "{{ abs_profile | default([profile_path | default(default_profile_path), profile | default(default_profile_file)] | path_join ) }}" init__definition_file: "{{ abs_definition | default( [definition_path, definition_file | default(default_definition_file)] | path_join ) }}" init__cluster_file: "{{ abs_cluster | default( [definition_path, cluster_file | default(default_cluster_file)] | path_join ) }}" - init__dynamic_inventory_template: "{{ abs_template | default( [definition_path, inventory_template | default(default_inventory_template)] | path_join ) }}" - init__dynamic_inventory_artefact: "{{ abs_inventory | default( [definition_path, static_inventory | default(default_static_inventory) ] | path_join ) }}" - init__download_mirror_artefact: "{{ download_mirror_file | default(download_mirror_file) }}" # Handle User Config - name: Check for User Config file @@ -216,7 +211,7 @@ - name: Check supplied Namespace when: - globals.infra_type != 'azure' - - "'teardown' not in {{ ansible_run_tags }}" + - "'teardown' not in ansible_run_tags" ansible.builtin.assert: that: - globals.name_prefix | length > 1 @@ -230,6 +225,7 @@ that: - globals.infra_deployment_engine in ['ansible', 'terraform'] fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'" + quiet: yes - name: Check Supplied terraform_base_dir variable when: @@ -309,68 +305,7 @@ - "SSH Private Key at {{ __private_key_file_stat.stat.path }} has invalid permissions" - "Permissions are {{ __private_key_file_stat.stat.mode }}" - "Permissions should be 0400 or 0600" - -# Parcel Distro -- name: Determine preferred Parcel Distribution - ansible.builtin.set_fact: - init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}" - -# Read in Dynamic Inventory -- name: Seek Inventory Template in Definition Path - register: __di_template_stat - ansible.builtin.stat: - path: "{{ init__dynamic_inventory_template }}" - -# inventory_dir is not defined when a user passes in an inventory with -i, so it is a useful check -# No point loading Dynamic Inventory if we are not doing infrastructure in this run -- name: Handle Dynamic Inventory Template - when: - - inventory_dir is defined - - __di_template_stat.stat.exists - block: - - name: Load in Dynamic Inventory Template - include_tasks: refresh_inventory.yml - vars: - include_inventory_file: "{{ __di_template_stat.stat.path }}" - - - name: Print Dynamic Inventory groups to debug at Verbosity 3 - debug: - msg: "{{ groups }}" - verbosity: 3 - - - name: Check expected minimum host groups appear in Inventory - ansible.builtin.assert: quiet: yes - that: - - groups.cluster is defined - - groups.cloudera_manager is defined - fail_msg: "Parsed Inventory Template did not contain minimum expected groups for a Cloudera Cluster deployment" - - - name: Check a custom_repo is part of the cluster definition if using Download Mirror - when: use_download_mirror | default(default_enable_download_mirror) | bool - ansible.builtin.assert: - that: "'custom_repo' in groups" - fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" - - - name: Extract list of hosts from Dynamic Inventory Template - when: groups.all | length > 0 - ansible.builtin.set_fact: - __dynamic_inventory_host_list: "{{ groups.all | difference(['localhost']) }}" - - - name: Set Dynamic Inventory host count in Globals - when: __dynamic_inventory_host_list | length > 0 - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __di_entry | default(omit), recursive=True ) }}" - loop_control: - loop_var: __di_entry - loop: - - dynamic_inventory: - vm: - count: "{{ __dynamic_inventory_host_list | count }}" - os: "{{ init__parcel_distro }}" - always: - - name: Remove Dynamic Inventory Template from current inventory - include_tasks: refresh_inventory.yml - name: Add no_log variables to globals no_log: true @@ -386,15 +321,6 @@ cloudera_manager_admin_password: "{{ globals.admin_password }}" no_log: true -# Download Mirror -- name: Prepare for Download Mirror Population if requested and necessary - when: - - use_download_mirror | default(default_enable_download_mirror) | bool - - "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: prepare_download_mirror - # Ansible tempfile doesn't appear to work well on ansible-runner - name: Set local Temp directory if not supplied ansible.builtin.set_fact: @@ -420,10 +346,10 @@ AWS_PROFILE: "{{ globals.aws_profile | default(omit) }}" AWS_REGION: "{{ globals.region | default(omit) }}" -- name: Print globals to debug at end of init (verbosity 3) +- name: Print globals to debug at end of init (verbosity 1) ansible.builtin.debug: msg: "{{ globals }}" - verbosity: 3 + verbosity: 1 - name: Determine if Cloud Roles should be called ansible.builtin.set_fact: @@ -438,46 +364,3 @@ fail_msg: >- Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars. quiet: yes - -# Provision Terraform remote state resources if requested -- name: Initialize Terraform remote state resources - when: - - init__call_cloud_role | bool - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - block: - # Set resource variable names if not already done - - name: Set variables for remote state bucket if not set - when: (globals.terraform.remote_state_bucket is not defined) or - ( (globals.terraform.remote_state_bucket) | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}" - - - name: Set variables for remote state lock table if not set - when: (globals.terraform.remote_state_lock_table is not defined) or - (globals.terraform.remote_state_lock_table | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}" - - - name: Create remote state resources - when: "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: auto_terraform_state - -- name: Prepare for Download Mirror Population if requested and necessary - when: - - use_download_mirror | default(default_enable_download_mirror) | bool - - "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: prepare_download_mirror \ No newline at end of file diff --git a/roles/cloudera_deploy/defaults/basic_cluster.yml b/roles/initialize_parameters/vars/basic_cluster.yml similarity index 100% rename from roles/cloudera_deploy/defaults/basic_cluster.yml rename to roles/initialize_parameters/vars/basic_cluster.yml diff --git a/roles/runlevels/tasks/main.yml b/roles/runlevels/tasks/main.yml new file mode 100644 index 0000000..338dae6 --- /dev/null +++ b/roles/runlevels/tasks/main.yml @@ -0,0 +1,43 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Configure runlevel tags + ansible.builtin.set_fact: + run_teardown: "{{ 'teardown' in ansible_run_tags }}" + run_infrastructure: "{{ 'infra' in ansible_run_tags }}" + run_platform: "{{ 'plat' in ansible_run_tags }}" + run_runtime: "{{ ansible_run_tags | difference(['infra', 'plat', 'teardown']) | length > 0 }}" + +- name: Check for supplied Definition path + ansible.builtin.assert: + quiet: yes + that: + - definition_path is defined + - definition_path | length > 0 + fail_msg: "You must supply a 'definition_path' referring to your definition directory" + +- name: Inspect the Definition path '{{ definition_path }}' + ansible.builtin.stat: + path: "{{ definition_path }}" + register: __definition_path_stat + +- name: Assert Definition path is a directory + ansible.builtin.assert: + quiet: yes + fail_msg: "'definition_path' does not refer to an existing and reachable definition directory" + that: + - __definition_path_stat.stat.isdir is defined + - __definition_path_stat.stat.isdir diff --git a/roles/terraform_state/NOTES.MD b/roles/terraform_state/NOTES.MD new file mode 100644 index 0000000..dbd642f --- /dev/null +++ b/roles/terraform_state/NOTES.MD @@ -0,0 +1,16 @@ +# USAGE + +```yml +- name: Teardown Cleanup + hosts: localhost + gather_facts: yes + tasks: + - name: Remove Terraform remote state resources if requested + when: + - globals.infra_deployment_engine == 'terraform' + - globals.terraform.auto_remote_state | bool + - globals.terraform.state_storage in ['remote_s3'] + ansible.builtin.include_role: + name: terraform_state + tasks_from: auto_terraform_state +``` \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/auto_terraform_state.yml b/roles/terraform_state/tasks/auto_terraform_state.yml similarity index 100% rename from roles/cloudera_deploy/tasks/auto_terraform_state.yml rename to roles/terraform_state/tasks/auto_terraform_state.yml diff --git a/roles/terraform_state/tasks/main.yml b/roles/terraform_state/tasks/main.yml new file mode 100644 index 0000000..b41f947 --- /dev/null +++ b/roles/terraform_state/tasks/main.yml @@ -0,0 +1,36 @@ +--- + +# Provision Terraform remote state resources if requested +- name: Initialize Terraform remote state resources + when: + - init__call_cloud_role | bool + - globals.infra_deployment_engine == 'terraform' + - globals.terraform.auto_remote_state | bool + - globals.terraform.state_storage in ['remote_s3'] + block: + # Set resource variable names if not already done + - name: Set variables for remote state bucket if not set + when: (globals.terraform.remote_state_bucket is not defined) or + ( (globals.terraform.remote_state_bucket) | length == 0) + ansible.builtin.set_fact: + globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" + vars: + remote_state_vars: + terraform: + remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}" + + - name: Set variables for remote state lock table if not set + when: (globals.terraform.remote_state_lock_table is not defined) or + (globals.terraform.remote_state_lock_table | length == 0) + ansible.builtin.set_fact: + globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" + vars: + remote_state_vars: + terraform: + remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}" + + - name: Create remote state resources + when: "'teardown' not in ansible_run_tags" + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: auto_terraform_state From 63bc11cd41d7bc0a3c38c0386c0d5bead6720e86 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 21 Jul 2022 21:57:17 +0100 Subject: [PATCH 2/5] Migrated roles from Cloudera-Deploy to cloudera.exe collection, as relative imports are difficult in Ansible, but FQCN imports are always good WIP migrating example definitions to separate repo Removed most uses of push-down Ansible tags as they confuse even the most experienced Ansible user Renamed several playbooks to be more explanatory of what service they work on Started splitting out playbooks into separate deployment areas to allow users to pick and choose, or run the whole thing from main.yml Updated private cloud cluster deployment to include necessary steps for ECS deployment, still WIP Refactored main to simplify some pathways for better readability, still WIP Signed-off-by: Daniel Chaffelson --- default/post_setup.yml | 5 - default/post_teardown.yml | 5 - default/pre_setup.yml | 5 - default/pre_teardown.yml | 5 - examples/refactor/definition.yml | 0 examples/refactor/post_setup.yml | 5 - examples/refactor/post_teardown.yml | 5 - examples/refactor/pre_setup.yml | 56 --- examples/refactor/pre_teardown.yml | 18 - examples/sandbox/definition.yml | 2 +- main.yml | 77 ++-- cloud_setup.yml => pbc_setup.yml | 0 cloud_teardown.yml => pbc_teardown.yml | 0 cluster_setup.yml => pvc_base_setup.yml | 183 ++++++--- cluster_teardown.yml => pvc_base_teardown.yml | 0 pvc_cp_setup.yml | 104 +++++ roles/download_mirror/NOTES.md | 35 -- .../tasks/inject_download_mirror.yml | 129 ------ roles/download_mirror/tasks/main.yml | 127 ------ .../tasks/populate_download_mirror.yml | 136 ------- .../tasks/prepare_download_mirror.yml | 19 - .../tasks/update_download_mirror_cache.yml | 31 -- roles/dynamic_inventory/NOTES.md | 45 --- roles/dynamic_inventory/defaults/main.yml | 5 - .../tasks/clean_dynamic_inventory.yml | 31 -- roles/dynamic_inventory/tasks/main.yml | 61 --- .../tasks/persist_dynamic_inventory.yml | 62 --- .../tasks/refresh_inventory.yml | 53 --- roles/initialize_parameters/NOTES.md | 15 - roles/initialize_parameters/defaults/main.yml | 55 --- .../tasks/distribute_facts_to_inventory.yml | 59 --- roles/initialize_parameters/tasks/main.yml | 366 ------------------ .../vars/basic_cluster.yml | 68 ---- roles/runlevels/tasks/main.yml | 43 -- roles/terraform_state/NOTES.MD | 16 - .../tasks/auto_terraform_state.yml | 48 --- roles/terraform_state/tasks/main.yml | 36 -- 37 files changed, 274 insertions(+), 1636 deletions(-) delete mode 100644 default/post_setup.yml delete mode 100644 default/post_teardown.yml delete mode 100644 default/pre_setup.yml delete mode 100644 default/pre_teardown.yml delete mode 100644 examples/refactor/definition.yml delete mode 100644 examples/refactor/post_setup.yml delete mode 100644 examples/refactor/post_teardown.yml delete mode 100644 examples/refactor/pre_setup.yml delete mode 100644 examples/refactor/pre_teardown.yml rename cloud_setup.yml => pbc_setup.yml (100%) rename cloud_teardown.yml => pbc_teardown.yml (100%) rename cluster_setup.yml => pvc_base_setup.yml (79%) rename cluster_teardown.yml => pvc_base_teardown.yml (100%) create mode 100644 pvc_cp_setup.yml delete mode 100644 roles/download_mirror/NOTES.md delete mode 100644 roles/download_mirror/tasks/inject_download_mirror.yml delete mode 100644 roles/download_mirror/tasks/main.yml delete mode 100644 roles/download_mirror/tasks/populate_download_mirror.yml delete mode 100644 roles/download_mirror/tasks/prepare_download_mirror.yml delete mode 100644 roles/download_mirror/tasks/update_download_mirror_cache.yml delete mode 100644 roles/dynamic_inventory/NOTES.md delete mode 100644 roles/dynamic_inventory/defaults/main.yml delete mode 100644 roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml delete mode 100644 roles/dynamic_inventory/tasks/main.yml delete mode 100644 roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml delete mode 100644 roles/dynamic_inventory/tasks/refresh_inventory.yml delete mode 100644 roles/initialize_parameters/NOTES.md delete mode 100644 roles/initialize_parameters/defaults/main.yml delete mode 100644 roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml delete mode 100644 roles/initialize_parameters/tasks/main.yml delete mode 100644 roles/initialize_parameters/vars/basic_cluster.yml delete mode 100644 roles/runlevels/tasks/main.yml delete mode 100644 roles/terraform_state/NOTES.MD delete mode 100644 roles/terraform_state/tasks/auto_terraform_state.yml delete mode 100644 roles/terraform_state/tasks/main.yml diff --git a/default/post_setup.yml b/default/post_setup.yml deleted file mode 100644 index 3edd289..0000000 --- a/default/post_setup.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default post-deploy set up - hosts: localhost - gather_facts: no diff --git a/default/post_teardown.yml b/default/post_teardown.yml deleted file mode 100644 index 7fee23e..0000000 --- a/default/post_teardown.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default post-deploy tear down - hosts: localhost - gather_facts: no diff --git a/default/pre_setup.yml b/default/pre_setup.yml deleted file mode 100644 index d3d3a84..0000000 --- a/default/pre_setup.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default pre-deploy set up - hosts: localhost - gather_facts: no diff --git a/default/pre_teardown.yml b/default/pre_teardown.yml deleted file mode 100644 index 4801ba4..0000000 --- a/default/pre_teardown.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default pre-deploy tear down - hosts: localhost - gather_facts: no diff --git a/examples/refactor/definition.yml b/examples/refactor/definition.yml deleted file mode 100644 index e69de29..0000000 diff --git a/examples/refactor/post_setup.yml b/examples/refactor/post_setup.yml deleted file mode 100644 index 3edd289..0000000 --- a/examples/refactor/post_setup.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default post-deploy set up - hosts: localhost - gather_facts: no diff --git a/examples/refactor/post_teardown.yml b/examples/refactor/post_teardown.yml deleted file mode 100644 index 7fee23e..0000000 --- a/examples/refactor/post_teardown.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: Default post-deploy tear down - hosts: localhost - gather_facts: no diff --git a/examples/refactor/pre_setup.yml b/examples/refactor/pre_setup.yml deleted file mode 100644 index 4614607..0000000 --- a/examples/refactor/pre_setup.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- - -- name: Example pre-deploy set up playbook - hosts: localhost - connection: local - gather_facts: no - tasks: - -# Inherits 'infra', 'plat', and 'run' from playbook import -# Applies 'validate' and 'initialize' via role import -# Data Service runtimes, like 'dw' and 'ml', can applied selectively -# within a role's tasks or via role import, as below - - - name: Start pre-deployment tasks - debug: - msg: This is the start. - - - name: Validate Infrastructure configuration - ansible.builtin.import_role: - name: cloudera.exe.infrastructure - tasks_from: validate - tags: - - validate - - de - - df - - dh - - dw - - ml - - opdb - - - name: Initialize Infrastructure setup - #when: sequence__setup_infra | bool - ansible.builtin.import_role: - name: cloudera.exe.infrastructure - tasks_from: initialize_setup - tags: - - initialize - - de - - df - - dh - - dw - - ml - - opdb - - - name: Set up Infrastructure - #when: sequence__setup_infra | bool - ansible.builtin.include_role: - name: cloudera.exe.infrastructure - tasks_from: setup - tags: - - de - - df - - dh - - dw - - ml - - opdb \ No newline at end of file diff --git a/examples/refactor/pre_teardown.yml b/examples/refactor/pre_teardown.yml deleted file mode 100644 index 5c7f698..0000000 --- a/examples/refactor/pre_teardown.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - -- name: Example pre-deploy tear down - hosts: localhost - gather_facts: no - tasks: - - # Inherits 'teardown' and 'never' from playbook import - - - name: Initialize Infrastructure teardown - ansible.builtin.import_role: - name: cloudera.exe.infrastructure - tasks_from: initialize_teardown - - - name: Tear down Infrastructure - ansible.builtin.include_role: - name: cloudera.exe.infrastructure - tasks_from: teardown diff --git a/examples/sandbox/definition.yml b/examples/sandbox/definition.yml index 797774c..ebc392e 100644 --- a/examples/sandbox/definition.yml +++ b/examples/sandbox/definition.yml @@ -19,5 +19,5 @@ datahub: - include: "datahub_streams_messaging_light.j2" use_default_cluster_definition: yes -use_download_mirror: yes +use_auto_repo_mirror: yes preload_cm_parcel_repo: yes \ No newline at end of file diff --git a/main.yml b/main.yml index 59b3c0f..61b1c33 100644 --- a/main.yml +++ b/main.yml @@ -25,7 +25,7 @@ # - teardown # # In this context, 'run' means 'runtime' or 'application'. The platform level -# is now both CDP Public Cloud SDX and Data Services, which is a change from +# is now both CDP Public Cloud SDX and Data Services, which is a change # from before where 'plat' referred to the DL and SDX and 'run' referred to the # Data Services. # @@ -38,51 +38,54 @@ # - ml # - opdb -- name: Establish Cloudera Deploy runlevels - hosts: localhost - gather_facts: no - run_once: True - tags: always - tasks: - - ansible.builtin.import_role: - name: runlevels - -- name: Initialize Cloudera Deploy parameters +- name: Initialize Cloudera Deploy Run hosts: localhost + connection: local gather_facts: yes tags: always tasks: - - ansible.builtin.import_role: - name: initialize_parameters # Should/could be more granular + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + +- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_teardown.yml'] | path_join }}" + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_teardown -- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'post_teardown.yml'] | path_join }}" - tags: ['never', 'plat', 'infra', 'teardown'] - when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_teardown +- ansible.builtin.import_playbook: pbc_teardown.yml + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + - hostvars.localhost.init__call_cdp_pbc == True -- ansible.builtin.import_playbook: cloud_teardown.yml - tags: ['never', 'infra', 'teardown'] - when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown +- ansible.builtin.import_playbook: pvc_base_teardown.yml + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown + - hostvars.localhost.init__call_cdp_pvc == True -- ansible.builtin.import_playbook: cluster_teardown.yml - tags: ['never', 'infra', 'teardown'] - when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_teardown +- ansible.builtin.import_playbook: "{{ [definition_path, 'pre_setup.yml'] | path_join }}" + when: + - hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_runtime -- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'pre_setup.yml'] | path_join }}" - tags: ['run', 'plat', 'infra'] - when: hostvars.localhost.run_infrastructure or hostvars.localhost.run_platform or hostvars.localhost.run_runtime +- ansible.builtin.import_playbook: pbc_setup.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pbc == True -- ansible.builtin.import_playbook: cloud_setup.yml - tags: ['run', 'plat'] - when: hostvars.localhost.run_platform or hostvars.localhost.run_runtime +- ansible.builtin.import_playbook: pvc_base_setup.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True -- ansible.builtin.import_playbook: cluster_setup.yml - tags: ['run', 'plat'] - when: hostvars.localhost.run_platform or hostvars.localhost.run_runtime +- ansible.builtin.import_playbook: pvc_cp_setup.yml + when: + - hostvars.localhost.run_pvc + - hostvars.localhost.init__call_cdp_pvc == True -- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'post_setup.yml'] | path_join }}" - tags: ['run'] - when: hostvars.localhost.run_runtime +- ansible.builtin.import_playbook: "{{ [definition_path, 'post_setup.yml'] | path_join }}" + when: + - hostvars.localhost.run_runtime -- ansible.builtin.import_playbook: "{{ [definition_path | default('default'), 'pre_teardown.yml'] | path_join }}" - tags: ['never', 'teardown'] - when: hostvars.localhost.run_teardown +- ansible.builtin.import_playbook: "{{ [definition_path, 'post_teardown.yml'] | path_join }}" + when: + - hostvars.localhost.run_teardown diff --git a/cloud_setup.yml b/pbc_setup.yml similarity index 100% rename from cloud_setup.yml rename to pbc_setup.yml diff --git a/cloud_teardown.yml b/pbc_teardown.yml similarity index 100% rename from cloud_teardown.yml rename to pbc_teardown.yml diff --git a/cluster_setup.yml b/pvc_base_setup.yml similarity index 79% rename from cluster_setup.yml rename to pvc_base_setup.yml index e2d69c7..94183c8 100644 --- a/cluster_setup.yml +++ b/pvc_base_setup.yml @@ -19,20 +19,20 @@ # This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml # STARTBLOCK # Init run -- name: Init Cluster Deployment Tasks +- name: Init PvC Base Deployment Tasks hosts: localhost - gather_facts: no + gather_facts: yes tasks: - # TODO: Move to dependent roles with delegation if necessary - - name: Create local temp directories - file: "{{ __dir }}" - loop: - - path: "{{ local_temp_dir }}/csrs" - state: directory - - path: "{{ local_temp_dir }}/certs" - state: directory - loop_control: - loop_var: __dir + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml tags: - always @@ -42,9 +42,6 @@ tasks: - name: Check connectivity to Inventory ansible.builtin.wait_for_connection: - - - name: Gather facts from connected inventory - setup: tags: - always @@ -108,7 +105,7 @@ # STARTBLOCK # Prepare Nodes - name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster, ca_server + hosts: cloudera_manager, cluster, ca_server, ecs_nodes become: yes roles: - cloudera.cluster.prereqs.os @@ -117,8 +114,26 @@ - default_cluster - full_cluster +- name: Apply OS Prereqs to ECS Nodes + hosts: ecs_nodes + gather_facts: yes + become: yes + tags: + - pvc + - os + - default_cluster + - full_cluster + tasks: + - name: Setup iptables or nftables + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.iptables + + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + - name: Create local user accounts - hosts: cloudera_manager, cluster, tls + hosts: cloudera_manager, cluster become: yes gather_facts: no roles: @@ -128,18 +143,36 @@ - default_cluster - full_cluster +- name: Create local users on ECS Nodes + hosts: ecs_nodes + gather_facts: yes + tags: + - pvc + - users + - default_cluster + - full_cluster + tasks: + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + - name: Install JDK - hosts: cloudera_manager, cluster, tls + hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes become: yes roles: - cloudera.cluster.prereqs.jdk tags: - jdk + - security + - free_ipa + - kerberos + - tls - default_cluster - full_cluster +# DB Connectors - name: Install MySQL Connector - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes gather_facts: no become: yes roles: @@ -150,7 +183,7 @@ - full_cluster - name: Install Oracle Connector - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes gather_facts: no become: yes roles: @@ -160,21 +193,23 @@ - oracle_connector - full_cluster -- name: Install PostgreSQL Connector - hosts: cloudera_manager, cluster - gather_facts: no +# ENDBLOCK # Prepare Nodes +# STARTBLOCK # Install Cluster Service Infrastructure II + +- name: Install RDBMS + hosts: db_server become: yes roles: - - role: cloudera.cluster.prereqs.postgresql_connector - when: database_type == 'postgresql' + - cloudera.cluster.infrastructure.rdbms tags: - - postgresql_connector + - database + - default_cluster - full_cluster -# ENDBLOCK # Prepare Nodes +# ENDBLOCK # Install Cluster Service Infrastructure II # STARTBLOCK # Create Cluster Service Infrastructure -- name: Install KDC +- name: Install Kerberos Server hosts: krb5_server become: yes roles: @@ -182,11 +217,12 @@ tags: - security - kerberos + - free_ipa - tls - full_cluster - name: Setup KRB5 clients - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes become: yes roles: - role: cloudera.cluster.infrastructure.krb5_client @@ -194,6 +230,7 @@ tags: - security - kerberos + - free_ipa - tls - full_cluster @@ -251,17 +288,6 @@ - always # ENDBLOCK # Prepare TLS -# STARTBLOCK # Install Cluster Service Infrastructure II -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - cloudera.cluster.infrastructure.rdbms - tags: - - database - - default_cluster - - full_cluster -# ENDBLOCK # Install Cluster Service Infrastructure II # STARTBLOCK # NiFi TLS - name: Setup symlinks for NiFi TLS keystore and truststore @@ -279,7 +305,7 @@ # STARTBLOCK # Install Cloudera Manager - name: Install Cloudera Manager daemons - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes become: yes any_errors_fatal: true roles: @@ -311,7 +337,7 @@ - full_cluster - name: Install Cloudera Manager agents - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes become: yes any_errors_fatal: true roles: @@ -334,7 +360,7 @@ - full_cluster - name: Configure Cloudera Manager agents - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes become: yes any_errors_fatal: true roles: @@ -364,6 +390,9 @@ - cm - full_cluster +# ENDBLOCK # Install Cloudera Manager +# STARTBLOCK # Cloudera Manager Password + - name: Configure Cloudera Manager Password hosts: cloudera_manager gather_facts: no @@ -374,9 +403,6 @@ - default_cluster - full_cluster -# ENDBLOCK # Install Cloudera Manager -# STARTBLOCK # Cloudera Manager Password - - name: Check Cloudera Manager admin password hosts: cloudera_manager, cluster gather_facts: no @@ -389,18 +415,19 @@ # ENDBLOCK # Cloudera Manager Password # STARTBLOCK # Prepare Security -# Insert AutoTLS with tag + - name: Enable Auto-TLS hosts: cloudera_manager gather_facts: no roles: - - cloudera.cluster.cloudera_manager.autotls + - role: cloudera.cluster.cloudera_manager.autotls + when: autotls is defined and autotls == True tags: - autotls - - never + - full_cluster - name: Install pre-requisite packages for Kerberos - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ecs_nodes become: yes roles: - role: cloudera.cluster.prereqs.kerberos @@ -421,14 +448,17 @@ - full_cluster # ENDBLOCK # Prepare Security -# STARTBLOCK # Install Cluster +# STARTBLOCK # Configure CM - name: Restart Cloudera Manager Agents hosts: cloudera_manager, cluster gather_facts: no become: yes - roles: - - role: cloudera.cluster.operations.restart_agents + tasks: + - name: Restart Cloudera Manager Agents + meta: noop + notify: + - restart cloudera-scm-agent tags: - never - restart_agents @@ -467,6 +497,9 @@ - default_cluster - full_cluster +# ENDBLOCK # Configure CM +# STARTBLOCK # Install Cluster + - name: Deploy clusters hosts: cloudera_manager gather_facts: no @@ -478,6 +511,19 @@ - full_cluster # ENDBLOCK # Install Cluster +# STARTBLOCK # Fix Auto-TLS + +- name: Auto-TLS Services Setup + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.cms_tls + when: autotls is defined and autotls == True + tags: + - autotls + - never + +# ENDBLOCK # Fix Auto-TLS # STARTBLOCK # Setup HDFS Encryption - name: Setup KTS HA @@ -530,7 +576,36 @@ - restart_stale - full_cluster -# ENDBLOCK # Setup HDFS Encryption + # ENDBLOCK # Setup HDFS Encryption + # STARTBLOCK # PVC Setup + +- name: Prepare Hosts for PvC Setup + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: yes + tags: + - pvc + tasks: + - name: Add missing ExtJS for Oozie UI + include_role: + name: cloudera.cluster.config.services.oozie_ui + when: oozie_service_exists | default(false) + +- name: Handle WXM Setup + hosts: "{{ tp_host | default('cluster_master_nodes[0]') }}" + gather_facts: yes + tags: + - wxm + - full_cluster + tasks: + - name: Setup WXM + when: + - use_wxm | default(False) + - altus_key_id | length > 0 + - altus_private_key | length > 0 + import_role: + name: cloudera.cluster.deployment.services.wxm + +# ENDBLOCK # PVC Setup # End run ### diff --git a/cluster_teardown.yml b/pvc_base_teardown.yml similarity index 100% rename from cluster_teardown.yml rename to pvc_base_teardown.yml diff --git a/pvc_cp_setup.yml b/pvc_cp_setup.yml new file mode 100644 index 0000000..4aee30c --- /dev/null +++ b/pvc_cp_setup.yml @@ -0,0 +1,104 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution +# This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml + +# STARTBLOCK # Init run +- name: Init PvC Base Deployment Tasks + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + tags: + - always + +- name: Check Inventory Connectivity + hosts: all + gather_facts: no + tasks: + - ansible.builtin.wait_for_connection: + tags: + - always + +# ENDBLOCK # Init run + # STARTBLOCK # PVC Setup +- name: Post-Install for PvC on all cluster hosts + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: yes + tags: + - pvc + tasks: + - name: Add missing ExtJS for Oozie UI + include_role: + name: cloudera.cluster.config.services.oozie_ui + when: oozie_service_exists | default(false) + +- name: Post-Install Cloudera Manager and Cluster + hosts: cloudera_manager + gather_facts: yes + tags: + - pvc + tasks: + - name: Refresh CM Services Info + include_role: + name: cloudera.cluster.cloudera_manager.services_info + public: yes + + - name: Fix Hue ticket lifetime for Free IPA + include_role: + name: cloudera.cluster.config.services.hue_ticket_lifetime + when: + - hue_service_exists | default(false) + - krb5_kdc_type == 'Red Hat IPA' + + - name: Set Cloudera Manager session timeout to 30d + include_role: + name: cloudera.cluster.cloudera_manager.session_timeout + + - name: Create missing Solr plugin for Ranger + include_role: + name: cloudera.cluster.config.services.solr_ranger_plugin + when: + - ranger_service_exists | default(false) + - solr_service_exists | default(false) + + - name: Add Solr urls to Knox + include_role: + name: cloudera.cluster.config.services.solr_knox + when: knox_service_exists | default(false) and solr_service_exists | default(false) + + - name: Add missing TLS values for KMS + when: + - kms_service_exists | default(False) + - (autotls | default(False)) + include_role: + name: cloudera.cluster.config.services.kms_tls + + - name: Ranger extra policies to have enough rights with basic accounts + when: ranger_service_exists | default(false) + include_role: + name: cloudera.cluster.config.services.ranger_pvc_default_policies + + +# ENDBLOCK # PVC Setup + +# End run +### diff --git a/roles/download_mirror/NOTES.md b/roles/download_mirror/NOTES.md deleted file mode 100644 index 7f7f38e..0000000 --- a/roles/download_mirror/NOTES.md +++ /dev/null @@ -1,35 +0,0 @@ -# USAGE - -```yml -- name: Prepare for Cloudera Cluster Run - hosts: localhost - tags: always - gather_facts: yes - tasks: - - name: Inject Download Mirror if requested - when: - - "'teardown' not in ansible_run_tags" - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: inject_download_mirror - -# TODO: Set timeout waiting for utility VM to be ready -- name: Process Download Mirror on Utility VM - hosts: cldr_utility - tags: always - gather_facts: yes - tasks: - - ansible.builtin.include_role: - name: download_mirror - tasks_from: prepare_download_mirror.yml - -- name: Update relevant Download Mirror Cache listing - hosts: localhost - tags: always - gather_facts: no - tasks: - - ansible.builtin.include_role: - name: download_mirror - tasks_from: update_download_mirror_cache.yml -``` \ No newline at end of file diff --git a/roles/download_mirror/tasks/inject_download_mirror.yml b/roles/download_mirror/tasks/inject_download_mirror.yml deleted file mode 100644 index f087f5b..0000000 --- a/roles/download_mirror/tasks/inject_download_mirror.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check if Download Mirror cache file exists - register: __download_mirror_file_stat - ansible.builtin.stat: - path: "{{ init__download_mirror_artefact }}" - -- name: Prepare for Download Mirror parsing - when: - - __download_mirror_file_stat.stat.exists - - use_download_mirror | default(default_enable_download_mirror) | bool - block: - - name: Check a custom_repo is part of the cluster definition if using Download Mirror - when: use_download_mirror | default(default_enable_download_mirror) | bool - ansible.builtin.assert: - that: "'custom_repo' in groups" - fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" - - - name: Handle AWS cache key generation - when: globals.infra_type == 'aws' - block: - - name: Get AWS Account Info - amazon.aws.aws_caller_info: - register: __aws_caller_info - - - name: Set parcel cache ini lookup key - ansible.builtin.set_fact: - init__download_mirror_ini_key: "{{ __aws_caller_info.account }}" - -# Ini lookup fails when section is not present, but lacks good control characteristics that I can find -- name: Determine if there is a relevant cache entry - when: - - init__download_mirror_ini_key is defined - - use_download_mirror | default(default_enable_download_mirror) | bool - ignore_errors: yes - ansible.builtin.set_fact: - __download_mirror_ini_entry: "{{ lookup('ini', __ini_lookup) }}" - vars: - __ini_lookup: ".+{{ init__download_mirror_ini_key }}.+ section={{ globals.infra_type }}:{{ globals.region }} file={{ init__download_mirror_artefact }} re=yes" - -- name: Handle Download Mirror injection if cache entry found - when: - - __download_mirror_ini_entry is defined - block: - - name: Generate a unique name - set_fact: - __tmp_cluster_file: "{{ ['/tmp', 99999999 | random | to_uuid] | path_join }}" - - - name: Copy Cluster definition to temp file - copy: - src: "{{ init__cluster_definition_file }}" - dest: "{{ __tmp_cluster_file }}" - - - name: Inject Parcel cache entries to Repository URLs - ansible.builtin.replace: - name: "{{ __tmp_cluster_file }}" - regexp: '^(\s+\-\s)https://archive\.cloudera\.com(\/.+)$' - replace: '\1http://{{ groups.custom_repo | first }}\2' - - - name: Set Cluster Definition file to Temp file with parcel cache entries - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ __tmp_cluster_file }}" - - - name: Fetch repositories from cluster definition - ansible.builtin.set_fact: - init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}" - - - name: Prepare lookup list of Repository entries - loop: "{{ init__preparse_repo_listing }}" - loop_control: - loop_var: __cluster_repo_item - ansible.builtin.set_fact: - init__cluster_repo_entries: "{{ init__cluster_repo_entries | default([]) + [__cluster_repo_item | urlsplit('path') ] }}" - - - name: Create list of Download Mirror URLs filtered to required repositories and distros - loop: "{{ init__cluster_repo_entries }}" - loop_control: - loop_var: __cluster_repo_path_item - ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __download_mirror_ini_entry - | select('search', __cluster_repo_path_item) - | select('search', init__parcel_distro) - | list }}" - - - name: Ensure manifest is included in Download Mirror URLs if present - loop: "{{ init__cluster_repo_entries }}" - loop_control: - loop_var: __cluster_repo_path_item - ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign - | default([]) + __download_mirror_ini_entry - | select('search', 'manifest.json') - | list }}" - -- name: Get AWS Specific download URIs - when: - - globals.infra_type == 'aws' - - init__urls_to_sign is defined - block: - - name: Generate signed URIs for hosted parcels to be pulled into custom_repo - register: __s3_signed_uris - loop: "{{ init__urls_to_sign }}" - loop_control: - loop_var: __s3_bucket_uri - amazon.aws.aws_s3: - bucket: "{{ __s3_bucket_uri | regex_replace('^.+//(.+)\\.s3.+$', '\\1') }}" - object: "{{ __s3_bucket_uri | regex_replace('^.+amazonaws\\.com(.+)$', '\\1') }}" - ignore_nonexistent_bucket: yes - expiry: "{{ download_link_expiry | default(default_download_link_expiry) }}" - mode: geturl - - - name: Set List of files to download to custom_repo - ansible.builtin.set_fact: - download_mirror_file_list: "{{ __s3_signed_uris.results | json_query('[*].url') | list }}" diff --git a/roles/download_mirror/tasks/main.yml b/roles/download_mirror/tasks/main.yml deleted file mode 100644 index c5c0e88..0000000 --- a/roles/download_mirror/tasks/main.yml +++ /dev/null @@ -1,127 +0,0 @@ ---- - -# Download Mirror -#- name: Prepare for Download Mirror Population if requested and necessary -# when: -# - use_download_mirror | default(default_enable_download_mirror) | bool -# - "'teardown' not in ansible_run_tags" -# ansible.builtin.include_role: -# name: cloudera_deploy -# tasks_from: prepare_download_mirror - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Read in cluster definition without jinja parsing -- name: Fetch repositories from cluster definition - ansible.builtin.set_fact: - init__preparse_repo_listing: "{{ lookup('file', init__cluster_definition_file ) | from_yaml | json_query('clusters[*].repositories') | flatten }}" - -- name: Check that a Cloudera License is presented if mirroring from files behind Cloudera Subscription - when: init__preparse_repo_listing | select('search', 'archive.cloudera.com/p') | length > 0 - ansible.builtin.assert: - that: - - globals.cloudera_license_file is defined - - globals.cloudera_license_file | length > 0 - fail_msg: "You must Supply a Cloudera License file to download and mirror files from archive.cloudera.com" - -- name: Separate direct Repos from tarballs and set Initial Facts - ansible.builtin.set_fact: - __init_parcel_repos: "{{ init__preparse_repo_listing | reject('search', 'tar.gz') | default([]) }}" - __init_tarball_links: "{{ init__preparse_repo_listing | select('search', 'tar.gz') | default([]) }}" - -# This sets 'manifests' on the calling host, and provides the repo login details -- name: Get Parcel Manifests - ansible.builtin.include_role: - name: cloudera.cluster.deployment.repometa - public: yes - vars: - repositories: "{{ __init_parcel_repos }}" - cluster_os_distribution: "{{ init__parcel_distro }}" - -- name: Extract Parcel URLs from Manifests - ansible.builtin.set_fact: - __parcel_urls: "{{ manifests.results | cloudera.cluster.extract_parcel_urls }}" - __parcel_distro_search_term: "{{ init__parcel_distro }}.parcel" - -- name: Filter Parcels by distro - ansible.builtin.set_fact: - __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}" - -- name: Prepare target Download Mirror listing with parcels and attendant files - when: __filtered_parcel_urls | length > 0 - loop: "{{ __filtered_parcel_urls }}" - loop_control: - loop_var: __filtered_parcel_item - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets | default(__init_tarball_links) + [__filtered_parcel_item, __filtered_parcel_item + '.sha1', __filtered_parcel_item.replace(__filtered_parcel_item | basename, 'manifest.json') ] }}" - -# This sets 'cloudera_manager_repo_url' on the calling host -- name: Determine Cloudera Manager Repo - ansible.builtin.include_role: - role: cloudera.cluster.cloudera_manager.repo - vars: - install_repo_on_host: no - clusters: [] - -- name: Add Cloudera Manager Repo to File Mirror list - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets + [cloudera_manager_repo_url | regex_replace('^(.+\\/(\\d\\.\\d\\.\\d)\\/)(\\w+)\\/.+$', '\\1' + 'repo-as-tarball/cm' + '\\2' + '-' + '\\3' + '.tar.gz')] }}" - -- name: Include CSDs if set - when: - - cloudera_manager_csds is defined - - cloudera_manager_csds | length > 0 - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets + cloudera_manager_csds }}" - -- name: Resolve Download Mirror for AWS - when: globals.infra_type == 'aws' - block: - - name: Get AWS Account Info - amazon.aws.aws_caller_info: - register: __aws_caller_info - - - name: Prepare Localised Download Mirror utility bucket name - ansible.builtin.set_fact: - init__download_mirror_bucket_name: "{{ utility_bucket_name | default([ download_mirror_prefix | default(default_download_mirror_prefix), __aws_caller_info.account, globals.region ] | join('-') ) }}" - - - name: List current target cache contents if any exist - register: __download_mirror_lookup_initial - failed_when: - - __download_mirror_lookup_initial.s3_keys is not defined - - "'cannot be found' not in __download_mirror_lookup_initial.msg" - amazon.aws.aws_s3: - bucket: "{{ init__download_mirror_bucket_name }}" - mode: list - - - name: Filter Files not already in mirror to be downloaded - when: __download_mirror_lookup_initial.s3_keys is defined - loop: "{{ __download_mirror_lookup_initial.s3_keys }}" - loop_control: - loop_var: __init_s3key_item - ansible.builtin.set_fact: - init__file_mirror_targets: "{{ init__file_mirror_targets | reject('match', '^.+' + __init_s3key_item + '$') | list }}" - -- name: Set Download Mirror details in Globals for Cloud run - when: - - init__file_mirror_targets | length > 0 - - init__download_mirror_bucket_name is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __download_mirror_spec, recursive=True ) }}" - vars: - __download_mirror_spec: - download_mirror_targets: "{{ init__file_mirror_targets }}" - utility_bucket_name: "{{ init__download_mirror_bucket_name }}" - create_utility_service: yes diff --git a/roles/download_mirror/tasks/populate_download_mirror.yml b/roles/download_mirror/tasks/populate_download_mirror.yml deleted file mode 100644 index c2681a5..0000000 --- a/roles/download_mirror/tasks/populate_download_mirror.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Ensure paths exist for File downloads to match required object structure [ will retry while utility VM boots ] - loop: "{{ globals.download_mirror_targets }}" - loop_control: - loop_var: __tmp_mirror_dir_item - ansible.builtin.file: - path: "/tmp/{{ globals.utility_bucket_name }}{{ __tmp_mirror_dir_item | urlsplit('path') | dirname }}" - state: directory - -- name: Request Async Download of Files to path structure - when: globals.download_mirror_targets - register: __download_mirror_rehost_results - loop: "{{ globals.download_mirror_targets }}" - loop_control: - loop_var: __mirror_fetch_item - async: 7200 - poll: 0 - ansible.builtin.get_url: - url: "{{ __mirror_fetch_item }}" - dest: "/tmp/{{ globals.utility_bucket_name }}{{ __mirror_fetch_item | urlsplit('path') }}" - url_username: "{{ cloudera_manager_repo_username | default(omit) }}" - url_password: "{{ cloudera_manager_repo_password | default(omit) }}" - -- name: Track async downloads to completion [ This may take up to an hour the first time for multi-gb Parcels ] - loop: "{{ __download_mirror_rehost_results.results }}" - loop_control: - loop_var: __download_async_item - register: __async_download_results - until: __async_download_results.finished is defined and __async_download_results.finished - delay: 30 - retries: 240 - async_status: - jid: "{{ __download_async_item.ansible_job_id }}" - failed_when: - - __download_async_item.failed == True - - __download_async_item.finished != 1 - -# Unpack parcel tarballs -- name: Unpack parcel tarballs for convenient use - loop: "{{ globals.download_mirror_targets | select('search', 'parcels.tar.gz') | list }}" - loop_control: - loop_var: __parcel_unpack_item - ansible.builtin.unarchive: - extra_opts: [ --strip-components=1 ] - remote_src: yes - src: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') }}" - dest: "/var/www/html{{ __parcel_unpack_item | urlsplit('path') | dirname }}" - keep_newer: yes - -- name: Upload Download Mirror for AWS to S3 - when: globals.infra_type == 'aws' - block: - # Prepare to sync cache dir to S3 - - name: Setup System Rhel8 - when: - - ansible_os_family == 'RedHat' - - ansible_distribution_major_version | int >= 8 - become: yes - ansible.builtin.package: - lock_timeout: 180 - name: "{{ __package_item }}" - update_cache: yes - state: present - loop_control: - loop_var: __package_item - loop: - - epel-release - - python3 - - - name: Setup system Rhel7 - when: - - ansible_os_family == 'RedHat' - - ansible_distribution_major_version | int < 8 - become: yes - ansible.builtin.package: - name: "{{ __package_item }}" - state: present - lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(180, omit) }}" - loop_control: - loop_var: __package_item - loop: - - epel-release - - python-pip - - - name: Setup system Debian - when: ansible_os_family == "Debian" - block: - - name: enable Debian Repos - become: yes - apt_repository: - repo: "{{ __repo_item }}" - loop_control: - loop_var: __repo_item - loop: - - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }} universe" - - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-updates universe" - - "deb http://security.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-security universe" - - - name: Install Pip on Debian - become: yes - ansible.builtin.apt: - update_cache: yes - name: python3-pip - state: present - - - name: Prepare host for s3 actions - become: yes - ansible.builtin.pip: - name: "{{ __pip_item }}" - loop_control: - loop_var: __pip_item - loop: - - futures - - "{{ (ansible_python_version[0] == '2') | ternary('boto3 >= 1.4.4,<1.18', 'boto3 >= 1.20.0') }}" - - - name: Sync downloaded Files paths to S3 cache bucket - become: yes - community.aws.s3_sync: - bucket: "{{ globals.utility_bucket_name }}" - file_root: "/tmp/{{ globals.utility_bucket_name }}" - permission: private diff --git a/roles/download_mirror/tasks/prepare_download_mirror.yml b/roles/download_mirror/tasks/prepare_download_mirror.yml deleted file mode 100644 index 5fce0ee..0000000 --- a/roles/download_mirror/tasks/prepare_download_mirror.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -- name: Fetch necessary variables from Ansible Controller - ansible.builtin.set_fact: - globals: "{{ hostvars['localhost']['globals'] }}" - init__download_mirror_artefact: "{{ hostvars['localhost']['init__download_mirror_artefact'] }}" - -- name: Prepare Cloudera Subscription Credentials - ansible.builtin.include_role: - name: cloudera.cluster.deployment.credential - when: globals.cloudera_license_file is defined - vars: - cloudera_manager_license_file: "{{ globals.cloudera_license_file }}" - -- name: Populate the Download Mirror with new files - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: populate_download_mirror - when: globals.download_mirror_targets | length > 0 diff --git a/roles/download_mirror/tasks/update_download_mirror_cache.yml b/roles/download_mirror/tasks/update_download_mirror_cache.yml deleted file mode 100644 index 6cd77c4..0000000 --- a/roles/download_mirror/tasks/update_download_mirror_cache.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -- name: Refresh Listing of target cache contents - when: - - init__download_mirror_bucket_name is defined - - "'teardown' not in ansible_run_tags" - register: __infra_download_mirror_listing - failed_when: - - __download_mirror_lookup_initial.s3_keys is not defined - - "'cannot be found' not in __download_mirror_lookup_initial.msg" - amazon.aws.aws_s3: - bucket: "{{ init__download_mirror_bucket_name }}" - mode: list - -- name: Prepare updated Download Mirror contents as URLs - when: __infra_download_mirror_listing.s3_keys is defined - loop: "{{ __infra_download_mirror_listing.s3_keys }}" - loop_control: - loop_var: __download_mirror_s3_urls_item - ansible.builtin.set_fact: - __download_mirror_url_listing: "{{ __download_mirror_url_listing | default([]) + [['https:/', init__download_mirror_bucket_name + '.s3.amazonaws.com', __download_mirror_s3_urls_item ] | join('/') ] }}" - -- name: Persist Download Mirror to Definition path - when: - - __download_mirror_url_listing is defined - - __download_mirror_url_listing | length > 0 - community.general.ini_file: - path: "{{ init__download_mirror_artefact }}" - section: "{{ globals.infra_type }}:{{ globals.region }}" - option: "{{ init__download_mirror_bucket_name }}" - value: "{{ __download_mirror_url_listing }}" \ No newline at end of file diff --git a/roles/dynamic_inventory/NOTES.md b/roles/dynamic_inventory/NOTES.md deleted file mode 100644 index dde0e52..0000000 --- a/roles/dynamic_inventory/NOTES.md +++ /dev/null @@ -1,45 +0,0 @@ -# USAGE - -```yml -- name: Prepare for Cloudera Cluster Run - hosts: localhost - tags: always - gather_facts: yes - tasks: - - name: Load Static Inventory file if present - when: mgmt is defined | bool - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: refresh_inventory - vars: - include_inventory_file: "{{ init__dynamic_inventory_artefact }}" - -- name: Marshal Cloud Deployment - hosts: localhost - environment: "{{ globals.env_vars }}" - gather_facts: yes - tasks: - - name: Persist Dynamic Inventory to Definition Path if exists - tags: always - when: - - infra__dynamic_inventory_host_entries is defined - - infra__dynamic_inventory_host_entries | length > 0 - - init__dynamic_inventory_template is defined - - init__dynamic_inventory_template | length > 0 - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: persist_dynamic_inventory - apply: - tags: always -``` - -```yml -- name: Teardown Cleanup - hosts: localhost - gather_facts: yes - tasks: - - name: Remove current Dynamic Inventory file from Definition Path if exists - ansible.builtin.include_role: - name: dynamic_inventory - tasks_from: clean_dynamic_inventory -``` \ No newline at end of file diff --git a/roles/dynamic_inventory/defaults/main.yml b/roles/dynamic_inventory/defaults/main.yml deleted file mode 100644 index e0a8263..0000000 --- a/roles/dynamic_inventory/defaults/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -init__dynamic_inventory_template: "{{ abs_template | default( [definition_path, inventory_template | default(default_inventory_template)] | path_join ) }}" -init__dynamic_inventory_artefact: "{{ abs_inventory | default( [definition_path, static_inventory | default(default_static_inventory) ] | path_join ) }}" -init__download_mirror_artefact: "{{ download_mirror_file | default(download_mirror_file) }}" diff --git a/roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml b/roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml deleted file mode 100644 index a4a5a4e..0000000 --- a/roles/dynamic_inventory/tasks/clean_dynamic_inventory.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check for a Static Inventory file - stat: - path: "{{ init__dynamic_inventory_artefact }}" - register: __inventory_static - -- name: Create a backup - when: __inventory_static.stat.exists - copy: - src: "{{ __inventory_static.stat.path }}" - dest: "{{ [init__dynamic_inventory_artefact | splitext | first, ansible_date_time.epoch] | join('.') }}" - -- name: Remove static inventory file - ansible.builtin.file: - path: "{{ init__dynamic_inventory_artefact }}" - state: absent \ No newline at end of file diff --git a/roles/dynamic_inventory/tasks/main.yml b/roles/dynamic_inventory/tasks/main.yml deleted file mode 100644 index a5deff4..0000000 --- a/roles/dynamic_inventory/tasks/main.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Parcel Distro -- name: Determine preferred Parcel Distribution - ansible.builtin.set_fact: - init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}" - -# Read in Dynamic Inventory -- name: Seek Inventory Template in Definition Path - register: __di_template_stat - ansible.builtin.stat: - path: "{{ init__dynamic_inventory_template }}" - -# inventory_dir is not defined when a user passes in an inventory with -i, so it is a useful check -# No point loading Dynamic Inventory if we are not doing infrastructure in this run -- name: Handle Dynamic Inventory Template - when: - - inventory_dir is defined - - __di_template_stat.stat.exists - block: - - name: Load in Dynamic Inventory Template - include_tasks: refresh_inventory.yml - vars: - include_inventory_file: "{{ __di_template_stat.stat.path }}" - - - name: Print Dynamic Inventory groups to debug at Verbosity 3 - debug: - msg: "{{ groups }}" - verbosity: 3 - - - name: Check expected minimum host groups appear in Inventory - ansible.builtin.assert: - quiet: yes - that: - - groups.cluster is defined - - groups.cloudera_manager is defined - fail_msg: "Parsed Inventory Template did not contain minimum expected groups for a Cloudera Cluster deployment" - - - name: Check a custom_repo is part of the cluster definition if using Download Mirror - when: use_download_mirror | default(default_enable_download_mirror) | bool - ansible.builtin.assert: - that: "'custom_repo' in groups" - fail_msg: "You must have a custom_repo in your Cluster Inventory when using Download Mirror" - - - name: Extract list of hosts from Dynamic Inventory Template - when: groups.all | length > 0 - ansible.builtin.set_fact: - __dynamic_inventory_host_list: "{{ groups.all | difference(['localhost']) }}" - - - name: Set Dynamic Inventory host count in Globals - when: __dynamic_inventory_host_list | length > 0 - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( __di_entry | default(omit), recursive=True ) }}" - loop_control: - loop_var: __di_entry - loop: - - dynamic_inventory: - vm: - count: "{{ __dynamic_inventory_host_list | count }}" - os: "{{ init__parcel_distro }}" - always: - - name: Remove Dynamic Inventory Template from current inventory - include_tasks: refresh_inventory.yml \ No newline at end of file diff --git a/roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml b/roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml deleted file mode 100644 index c310169..0000000 --- a/roles/dynamic_inventory/tasks/persist_dynamic_inventory.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Generate a unique name - ansible.builtin.set_fact: - __tmp_dynamic_inventory_artefact: "{{ [inventory_dir, 99999999 | random | to_uuid] | path_join }}" - -- name: Copy Host Inventory Template to Temporary Static Inventory Artefact - ansible.builtin.copy: - src: "{{ init__dynamic_inventory_template }}" - dest: "{{ __tmp_dynamic_inventory_artefact }}" - -- name: Update Static Inventory Artefact with Host Entries from Cloud Infrastructure Run - loop: "{{ __dynamic_inventory_host_list | zip( infra__dynamic_inventory_host_entries ) }}" - loop_control: - loop_var: __infra_inventory_compute_item - ansible.builtin.replace: - name: "{{ __tmp_dynamic_inventory_artefact }}" - regexp: '(\s+){{ __infra_inventory_compute_item.0 }}(\s+)' - replace: '\1{{ __infra_inventory_compute_item.1 }}\2' - -- name: Stat the Temporary Artefact - ansible.builtin.stat: - path: "{{ __tmp_dynamic_inventory_artefact }}" - register: __tmp_inventory_static - -- name: Check for an existing Dynamic Inventory Artefact file - ansible.builtin.stat: - path: "{{ init__dynamic_inventory_artefact }}" - register: __inventory_static - -- name: Create a backup if the files are different - when: - - __inventory_static.stat.exists - - __tmp_inventory_static.stat.checksum != __inventory_static.stat.checksum - ansible.builtin.copy: - src: "{{ __inventory_static.stat.path }}" - dest: "{{ [init__dynamic_inventory_artefact | splitext | first, ansible_date_time.epoch] | join('.') }}" - -- name: Copy Temporary Dynamic Inventory Artefact to Inventory Artefact - ansible.builtin.copy: - src: "{{ __tmp_inventory_static.stat.path }}" - dest: "{{ init__dynamic_inventory_artefact }}" - force: yes - -- name: Remove Temporary Artefact file - ansible.builtin.file: - path: "{{ __tmp_inventory_static.stat.path }}" - state: absent diff --git a/roles/dynamic_inventory/tasks/refresh_inventory.yml b/roles/dynamic_inventory/tasks/refresh_inventory.yml deleted file mode 100644 index ce163fc..0000000 --- a/roles/dynamic_inventory/tasks/refresh_inventory.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO Consider converting to a handler? - -- name: Check for additional inventory file - ansible.builtin.stat: - path: "{{ include_inventory_file }}" - register: __add_inventory_static - -- name: Generate a unique name - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.set_fact: - __tmp_add_inventory_file: "{{ [inventory_dir, 99999999 | random | to_uuid] | path_join }}" - -- name: Temporarily copy Additional Inventory to Ansible inventory dir {{ inventory_dir }} - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.copy: - src: "{{ __add_inventory_static.stat.path }}" - dest: "{{ __tmp_add_inventory_file }}" - -- name: Refresh inventory - meta: refresh_inventory - -- name: Print updated inventory to log - ansible.builtin.debug: - msg: "{{ groups }}" - verbosity: 3 - -- name: Remove temporary static inventory file - when: - - __add_inventory_static.stat.exists - - inventory_dir is defined - ansible.builtin.file: - path: "{{ __tmp_add_inventory_file }}" - state: absent diff --git a/roles/initialize_parameters/NOTES.md b/roles/initialize_parameters/NOTES.md deleted file mode 100644 index a9d9e61..0000000 --- a/roles/initialize_parameters/NOTES.md +++ /dev/null @@ -1,15 +0,0 @@ -# USAGE - -```yml -- name: Prepare for Cloudera Cluster Run - hosts: localhost - tags: always - gather_facts: yes - tasks: - - name: Distribute Facts to Inventory Hosts - when: - - "'cluster' in groups" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: distribute_facts_to_inventory -``` \ No newline at end of file diff --git a/roles/initialize_parameters/defaults/main.yml b/roles/initialize_parameters/defaults/main.yml deleted file mode 100644 index 9715d25..0000000 --- a/roles/initialize_parameters/defaults/main.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default Paths -default_local_temp_dir: '/tmp' -default_sshkey_path: '~/.ssh' -default_config_path: '~/.config/cloudera-deploy' - -# Default names -default_name_prefix: cldr -default_ssh_key_suffix: _ssh_rsa -default_download_mirror_prefix: cache - -# Default Basic Cluster Config -default_cluster_definition_file: "basic_cluster.yml" - -# Default Artefact Filenames -default_profile_path: "{{ [default_config_path, 'profiles'] | path_join }}" -default_profile_file: "default" -default_definition_file: "definition.yml" -default_cluster_file: "cluster.yml" -default_inventory_template: "inventory_template.ini" -default_static_inventory: "inventory_static.ini" -download_mirror_file: "{{ [default_config_path, 'download_mirror.ini'] | path_join }}" - -include_inventory_file: '' - -# Default behavior -use_default_cluster_definition: no - -# Default parcel cache -default_enable_download_mirror: no -default_parcel_distro: el7 # el8, bionic, focal -default_download_link_expiry: 3600 - -# Default Deployment Controls -default_infra_deployment_engine: ansible -default_infra_type: aws -default_infra_region: us-east-1 - -# Terraform defaults -default_terraform_base_dir: "{{ [default_config_path, 'terraform'] | path_join }}" \ No newline at end of file diff --git a/roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml b/roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml deleted file mode 100644 index 4c74f6b..0000000 --- a/roles/initialize_parameters/tasks/distribute_facts_to_inventory.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Set specific Facts for later use in Cluster Deployment - ansible.builtin.set_fact: - _pre_template_cluster: "{{ lookup('file', init__cluster_definition_file ) | from_yaml }}" - preload_parcels: "{{ download_mirror_file_list | default([]) }}" - custom_repo_rehost_files: "{{ download_mirror_file_list | default([]) }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label: __play_host - -- name: Set Sensitive Facts with no-log for later use in Cluster Deployment - ansible.builtin.set_fact: - cloudera_manager_admin_password: "{{ globals.admin_password }}" - cloudera_manager_license_file: "{{ globals.cloudera_license_file | default(omit) }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - no_log: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label: __play_host - -- name: Include Definition Vars to allow overrides to propagate to Cluster Deployment - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label : __play_host - -- name: Include Cluster Definition override - ansible.builtin.include_vars: - file: "{{ init__cluster_definition_file }}" - delegate_to: "{{ __play_host }}" - delegate_facts: true - loop: "{{ groups.all }}" - loop_control: - loop_var: __play_host - label : __play_host \ No newline at end of file diff --git a/roles/initialize_parameters/tasks/main.yml b/roles/initialize_parameters/tasks/main.yml deleted file mode 100644 index 2ac9f98..0000000 --- a/roles/initialize_parameters/tasks/main.yml +++ /dev/null @@ -1,366 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Check versions -- name: Get Python packages - register: __python_env_packages - command: pip freeze - -- name: Get Ansible Collections - register: __ansible_collection_list - command: ansible-galaxy collection list - -- name: Get Ansible Roles - register: __ansible_role_list - command: ansible-galaxy role list - -- name: Log Runtime information - ansible.builtin.debug: - msg: - - "Ansible Version: {{ ansible_version | to_yaml }}" - - "Ansible Collections: {{ __ansible_collection_list.stdout | to_nice_yaml }}" - - "Ansible Roles: {{ __ansible_role_list.stdout }}" - - "Python Version: {{ ansible_python_version }}" - - "Python Packages: {{ __python_env_packages.stdout }}" - - "Runner Version: '{{ lookup('env', 'CLDR_BUILD_VER') }}'" - verbosity: 1 - -# Check path -- name: Check a Definition path has been supplied - ansible.builtin.assert: - quiet: yes - that: - - definition_path is defined - - definition_path | length > 0 - fail_msg: "You must supply a 'definition_path' pointing to your artefacts directory" - -- name: Check if Definition Path '{{ definition_path }}' exists - ansible.builtin.stat: - path: "{{ definition_path }}" - register: __definition_path_stat - -- name: Assert Definition Path is a directory - ansible.builtin.assert: - quiet: yes - fail_msg: "'definition_path' does not appear to point to an existing and reachable directory'" - that: - - __definition_path_stat.stat.isdir is defined - - __definition_path_stat.stat.isdir - -# Set File Paths -- name: Set Expected File Paths - ansible.builtin.set_fact: - init__user_profile: "{{ abs_profile | default([profile_path | default(default_profile_path), profile | default(default_profile_file)] | path_join ) }}" - init__definition_file: "{{ abs_definition | default( [definition_path, definition_file | default(default_definition_file)] | path_join ) }}" - init__cluster_file: "{{ abs_cluster | default( [definition_path, cluster_file | default(default_cluster_file)] | path_join ) }}" - -# Handle User Config -- name: Check for User Config file - register: __user_config_stat - ansible.builtin.stat: - path: "{{ init__user_profile }}" - -- name: Load User Config - when: __user_config_stat.stat.exists - ansible.builtin.include_vars: - file: "{{ __user_config_stat.stat.path }}" - -# Handle Definition File -- name: Seek Definition files in Definition Path - register: __def_file_stat - ansible.builtin.stat: - path: "{{ init__definition_file }}" - -- name: Assert that a Definition File has been provided - ansible.builtin.assert: - quiet: yes - that: __def_file_stat.stat.exists - fail_msg: "Expected to find a definition file '{{ init__definition_file }}' in Definition Path '{{ definition_path }}'" - -# User Definition files may contain lazy templating which would break if pre-merged here, therefore the cluster defaults file is kept separate -- name: Select default provided User Definition Files - ansible.builtin.set_fact: - init__user_definition_file: "{{ __def_file_stat.stat.path }}" - init__cluster_definition_file: "{{ __def_file_stat.stat.path }}" - -# Handle separate Cluster File -- name: Seek Cluster Definition files in Definition Path - register: __clus_file_stat - ansible.builtin.stat: - path: "{{ init__cluster_file }}" - -- name: Include vars to top level for other facts in Definition File - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - -# Must be included before cluster definition checks as it may have logic control switches -- name: Override with separate Cluster file if provided - when: __clus_file_stat.stat.exists - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ __clus_file_stat.stat.path }}" - -# Override with default cluster definition if requested, regardless of files found -- name: Use default cluster definition as override if requested - when: use_default_cluster_definition | bool - ansible.builtin.set_fact: - init__cluster_definition_file: "{{ default_cluster_definition_file }}" - -- name: Include vars from User Definition File to private dict to check for Globals - ansible.builtin.include_vars: - file: "{{ init__user_definition_file }}" - name: __def_vars - -# Note that this depends on the earlier set_fact for globals to take precedence over include_vars -- name: Include Cluster definition file for current localhost use after User Definition is Loaded - ansible.builtin.include_vars: - file: "{{ init__cluster_definition_file }}" - -- name: If Purge is defined, check it is boolean - when: purge is defined - ansible.builtin.assert: - that: purge|bool is sameas true or purge|bool is sameas false - fail_msg: "purge key is present in definition, but not a boolean as expected" - quiet: yes - -# Admin Password -- name: Prompt User for a password if not provided in config or vault - when: admin_password is undefined or admin_password | length < 2 - block: - - name: Prompt User for Password if not supplied - no_log: true - pause: - prompt: "No admin password found in profile.yml or extra_vars, or provided password too short; please provide a Password" - register: __user_input_password - - - name: Set Admin password - no_log: true - ansible.builtin.set_fact: - admin_password: "{{ __user_input_password.user_input }}" - -- name: Assert user has supplied an Admin Password - no_log: true - ansible.builtin.assert: - quiet: yes - that: - - admin_password is defined - - admin_password | length > 2 - fail_msg: "You must supply an Admin Password of at least 2 chars" - -# Merge User Profile to Globals -- name: Marshal User Config into Globals - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(user_config , recursive=True) }}" - vars: - user_config: - name_prefix: "{{ name_prefix | default(default_name_prefix) }}" - tags: "{{ tags | default(omit) }}" - region: "{{ infra_region | default(default_infra_region) }}" - infra_deployment_engine: "{{ infra_deployment_engine | default(default_infra_deployment_engine) }}" - infra_type: "{{ infra_type | default(default_infra_type) }}" - terraform: - base_dir: "{{ terraform.base_dir | default(default_terraform_base_dir) | expanduser }}" - state_storage: "{{ terraform.state_storage | default(omit) }}" - auto_remote_state: "{{ terraform.auto_remote_state | default(False) }}" - remote_state_bucket: "{{ terraform.remote_state_bucket | default(omit) }}" - remote_state_lock_table: "{{ terraform.remote_state_lock_table | default(omit) }}" - ssh: - public_key_id: "{{ public_key_id | default(omit) }}" - public_key_file: "{{ public_key_file | default(omit) }}" - public_key_text: "{{ public_key_text | default(omit) }}" - private_key_file: "{{ private_key_file | default(omit) }}" - key_path: "{{ ssh_key_path | default(default_sshkey_path) }}" - cloudera_license_file: "{{ license_file | default(omit) }}" - gcloud_credential_file: "{{ gcloud_credential_file | default(omit) }}" - cdp_profile: "{{ cdp_profile | default(omit) }}" - cdp_region: "{{ cdp_region | default(omit) }}" - aws_profile: "{{ aws_profile | default(omit) }}" - force_teardown: "{{ purge | default(omit) }}" - env_vars: "{{ env_vars | default(omit) }}" - -- name: Merge overwrite globals from Definition file with Globals on User File - when: __def_vars.globals is defined - ansible.builtin.set_fact: - globals: "{{ globals | combine(__def_vars.globals, recursive=True) }}" - -# Validate Name Prefix -- name: Check supplied Namespace (Azure) - when: - - globals.infra_type == 'azure' - - "'teardown' not in {{ ansible_run_tags }}" - ansible.builtin.assert: - that: - - globals.name_prefix | length > 1 - - globals.name_prefix | length < 5 - - globals.name_prefix | regex_search('^[a-zA-Z]') - fail_msg: "You must supply a valid Namespace" - quiet: yes - -- name: Check supplied Namespace - when: - - globals.infra_type != 'azure' - - "'teardown' not in ansible_run_tags" - ansible.builtin.assert: - that: - - globals.name_prefix | length > 1 - - globals.name_prefix | length < 8 - - globals.name_prefix | regex_search('^[a-zA-Z]') - fail_msg: "You must supply a valid Namespace" - quiet: yes - -- name: Check Deployment Engine variable - ansible.builtin.assert: - that: - - globals.infra_deployment_engine in ['ansible', 'terraform'] - fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'" - quiet: yes - -- name: Check Supplied terraform_base_dir variable - when: - - globals.infra_deployment_engine == 'terraform' - ansible.builtin.assert: - that: - - globals.terraform.base_dir is defined - - globals.terraform.base_dir | length > 0 - fail_msg: "You must supply a 'terraform_base_dir' where Terraform assets will be placed" - quiet: yes - -- name: Check Supplied terraform_auto_remote_state variable - when: - - globals.infra_deployment_engine == 'terraform' - ansible.builtin.assert: - that: - - (globals.terraform.auto_remote_state|bool is sameas true) or (globals.terraform.auto_remote_state|bool is sameas false) - fail_msg: "The terraform.auto_remote_state variable must be a boolean variable" - quiet: yes - -# SSH -- name: Use default SSH public key id - when: globals.ssh.public_key_id is undefined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__default_ssh_key_id, recursive=True) }}" - vars: - __default_ssh_key_id: - ssh: - public_key_id: "{{ globals.name_prefix }}" - -- name: Generate SSH public and private keys - when: (globals.ssh.public_key_file is undefined or globals.ssh.public_key_file | length < 3) and globals.ssh.public_key_text is undefined - block: - - name: Generate a SSH keypair (public and private keys) - register: __generated_ssh_keys - community.crypto.openssh_keypair: - path: "{{ [default_sshkey_path, __generated_keypair_name] | path_join }}" - comment: "{{ globals.name_prefix }} (auto-generated)" - type: rsa - size: 4096 - regenerate: never - force: no - vars: - __generated_keypair_name: "{{ globals.name_prefix + default_ssh_key_suffix }}" - - - name: Set facts for the generated SSH keypair details - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__generated_keypair, recursive=True) }}" - vars: - __generated_keypair: - ssh: - private_key_file: "{{ __generated_ssh_keys.filename }}" - public_key_file: "{{ [ __generated_ssh_keys.filename, 'pub' ] | join('.') }}" - -- name: Load SSH public key file to text - when: globals.ssh.public_key_file is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__public_key_globals , recursive=True) }}" - vars: - __public_key_globals: - ssh: - public_key_text: "{{ lookup('file', globals.ssh.public_key_file ) | default(omit) }}" - -- name: Validate SSH Private Key File has acceptable permissions - when: globals.ssh.private_key_file is defined - block: - - name: Get information for SSH Private Key File - ansible.builtin.stat: - path: "{{ globals.ssh.private_key_file }}" - register: __private_key_file_stat - - - name: Assert that SSH Private Key has valid permissions - ansible.builtin.assert: - that: - - __private_key_file_stat.stat.mode == '0400' or __private_key_file_stat.stat.mode == '0600' - fail_msg: - - "SSH Private Key at {{ __private_key_file_stat.stat.path }} has invalid permissions" - - "Permissions are {{ __private_key_file_stat.stat.mode }}" - - "Permissions should be 0400 or 0600" - quiet: yes - -- name: Add no_log variables to globals - no_log: true - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(__no_log_globals, recursive=True) }}" - vars: - __no_log_globals: - admin_password: "{{ admin_password | mandatory }}" - -- name: Set CM facts - ansible.builtin.set_fact: - cloudera_manager_license_file: "{{ globals.cloudera_license_file | default(omit) }}" - cloudera_manager_admin_password: "{{ globals.admin_password }}" - no_log: true - -# Ansible tempfile doesn't appear to work well on ansible-runner -- name: Set local Temp directory if not supplied - ansible.builtin.set_fact: - local_temp_dir: "{{ local_temp_dir | default(default_local_temp_dir) }}" - -- name: Set GCloud Environment Variables if needed - when: globals.gcloud_credential_file is defined - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_gcp_entries, recursive=True ) }}" - vars: - env_gcp_entries: - env_vars: - GCP_AUTH_KIND: serviceaccount - GCP_SERVICE_ACCOUNT_FILE: "{{ globals.gcloud_credential_file }}" - -- name: Set Profile Env Vars if required - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine( env_var_entries, recursive=True ) }}" - vars: - env_var_entries: - env_vars: - CDP_PROFILE: "{{ globals.cdp_profile | default(omit) }}" - AWS_PROFILE: "{{ globals.aws_profile | default(omit) }}" - AWS_REGION: "{{ globals.region | default(omit) }}" - -- name: Print globals to debug at end of init (verbosity 1) - ansible.builtin.debug: - msg: "{{ globals }}" - verbosity: 1 - -- name: Determine if Cloud Roles should be called - ansible.builtin.set_fact: - init__call_cloud_role: "{{ infra is defined or env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" - init__call_cdp_public: "{{ env is defined or ml is defined or de is defined or datahub is defined or opdb is defined or dw is defined or df is defined | default(False) }}" - -- name: Check Admin Password is CDP Public compliant when calling CDP Public - when: init__call_cdp_public | bool - ansible.builtin.assert: - that: - - admin_password is match('^(?=.*[A-Za-z])(?=.*\\d)(?=.*[@$!%*#?&])[A-Za-z\\d@$!%*#?&]{8,64}$') - fail_msg: >- - Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars. - quiet: yes diff --git a/roles/initialize_parameters/vars/basic_cluster.yml b/roles/initialize_parameters/vars/basic_cluster.yml deleted file mode 100644 index 752963b..0000000 --- a/roles/initialize_parameters/vars/basic_cluster.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cloudera_manager_version: 7.4.4 - -clusters: - - name: Basic Cluster - services: [HDFS, YARN, ZOOKEEPER] - repositories: - - https://archive.cloudera.com/cdh7/7.1.7.0/parcels/ - configs: - HDFS: - DATANODE: - dfs_data_dir_list: /dfs/dn - NAMENODE: - dfs_name_dir_list: /dfs/nn - SECONDARYNAMENODE: - fs_checkpoint_dir_list: /dfs/snn - YARN: - RESOURCEMANAGER: - yarn_scheduler_maximum_allocation_mb: 4096 - yarn_scheduler_maximum_allocation_vcores: 4 - NODEMANAGER: - yarn_nodemanager_resource_memory_mb: 4096 - yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm - yarn_nodemanager_log_dirs: /var/log/nm - GATEWAY: - mapred_submit_replication: 3 - mapred_reduce_tasks: 6 - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - host_templates: - Master1: - HDFS: [NAMENODE, SECONDARYNAMENODE] - YARN: [RESOURCEMANAGER, JOBHISTORY] - ZOOKEEPER: [SERVER] - Workers: - HDFS: [DATANODE] - YARN: [NODEMANAGER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true \ No newline at end of file diff --git a/roles/runlevels/tasks/main.yml b/roles/runlevels/tasks/main.yml deleted file mode 100644 index 338dae6..0000000 --- a/roles/runlevels/tasks/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- - -# Copyright 2022 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Configure runlevel tags - ansible.builtin.set_fact: - run_teardown: "{{ 'teardown' in ansible_run_tags }}" - run_infrastructure: "{{ 'infra' in ansible_run_tags }}" - run_platform: "{{ 'plat' in ansible_run_tags }}" - run_runtime: "{{ ansible_run_tags | difference(['infra', 'plat', 'teardown']) | length > 0 }}" - -- name: Check for supplied Definition path - ansible.builtin.assert: - quiet: yes - that: - - definition_path is defined - - definition_path | length > 0 - fail_msg: "You must supply a 'definition_path' referring to your definition directory" - -- name: Inspect the Definition path '{{ definition_path }}' - ansible.builtin.stat: - path: "{{ definition_path }}" - register: __definition_path_stat - -- name: Assert Definition path is a directory - ansible.builtin.assert: - quiet: yes - fail_msg: "'definition_path' does not refer to an existing and reachable definition directory" - that: - - __definition_path_stat.stat.isdir is defined - - __definition_path_stat.stat.isdir diff --git a/roles/terraform_state/NOTES.MD b/roles/terraform_state/NOTES.MD deleted file mode 100644 index dbd642f..0000000 --- a/roles/terraform_state/NOTES.MD +++ /dev/null @@ -1,16 +0,0 @@ -# USAGE - -```yml -- name: Teardown Cleanup - hosts: localhost - gather_facts: yes - tasks: - - name: Remove Terraform remote state resources if requested - when: - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - ansible.builtin.include_role: - name: terraform_state - tasks_from: auto_terraform_state -``` \ No newline at end of file diff --git a/roles/terraform_state/tasks/auto_terraform_state.yml b/roles/terraform_state/tasks/auto_terraform_state.yml deleted file mode 100644 index 9e60f32..0000000 --- a/roles/terraform_state/tasks/auto_terraform_state.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- - -# Copyright 2021 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Resources for remote_s3 state storage - when: - - globals.terraform.state_storage == 'remote_s3' - block: - - # Create or Teardown the resources - - name: AWS Bucket for Remote State Storage - amazon.aws.aws_s3: - region: "{{ globals.region }}" - bucket: "{{ globals.terraform.remote_state_bucket}}" - mode: "{{ ('teardown' not in ansible_run_tags) | ternary('create', 'delete') }}" # Check ansible tag to determine action - permission: private - register: __infra_aws_storage_locations_info - - - name: AWS DynamoDB for Remote State Locking - community.aws.dynamodb_table: - region: "{{ globals.region }}" - name: "{{ globals.terraform.remote_state_lock_table }}" - read_capacity: 1 - write_capacity: 1 - hash_key_name: LockID - hash_key_type: STRING - state: "{{ ('teardown' not in ansible_run_tags) | ternary('present', 'absent') }}" # Check ansible tag to determine action - - - name: Print remote state configuration - when: "'teardown' not in ansible_run_tags" - ansible.builtin.debug: - msg: - - "Resources for remote_s3 Terraform State created." - - "S3 Bucket Name: {{ globals.terraform.remote_state_bucket}}" - - "DynamoDB Locking Table: {{ globals.terraform.remote_state_lock_table}}" - verbosity: 3 \ No newline at end of file diff --git a/roles/terraform_state/tasks/main.yml b/roles/terraform_state/tasks/main.yml deleted file mode 100644 index b41f947..0000000 --- a/roles/terraform_state/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - -# Provision Terraform remote state resources if requested -- name: Initialize Terraform remote state resources - when: - - init__call_cloud_role | bool - - globals.infra_deployment_engine == 'terraform' - - globals.terraform.auto_remote_state | bool - - globals.terraform.state_storage in ['remote_s3'] - block: - # Set resource variable names if not already done - - name: Set variables for remote state bucket if not set - when: (globals.terraform.remote_state_bucket is not defined) or - ( (globals.terraform.remote_state_bucket) | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}" - - - name: Set variables for remote state lock table if not set - when: (globals.terraform.remote_state_lock_table is not defined) or - (globals.terraform.remote_state_lock_table | length == 0) - ansible.builtin.set_fact: - globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" - vars: - remote_state_vars: - terraform: - remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}" - - - name: Create remote state resources - when: "'teardown' not in ansible_run_tags" - ansible.builtin.include_role: - name: cloudera_deploy - tasks_from: auto_terraform_state From 899f36c6b58124e3ba9b535ec1007ca957ee98ba Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Sat, 23 Jul 2022 12:53:41 +0100 Subject: [PATCH 3/5] Rename iptables role to be generic prereqs for pvc_ecs Deploying a cluster requires facts to be gathered for ansible version, this Play now gathers facts Updated pvc_cp_setup.yml so it'll run independently of main.yml Signed-off-by: Daniel Chaffelson --- pvc_base_setup.yml | 4 ++-- pvc_cp_setup.yml | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pvc_base_setup.yml b/pvc_base_setup.yml index 94183c8..ca94828 100644 --- a/pvc_base_setup.yml +++ b/pvc_base_setup.yml @@ -126,7 +126,7 @@ tasks: - name: Setup iptables or nftables ansible.builtin.include_role: - name: cloudera.cluster.prereqs.iptables + name: cloudera.cluster.prereqs.pvc_ecs - name: Create user accounts for ECS nodes ansible.builtin.include_role: @@ -502,7 +502,7 @@ - name: Deploy clusters hosts: cloudera_manager - gather_facts: no + gather_facts: yes roles: - cloudera.cluster.deployment.cluster tags: diff --git a/pvc_cp_setup.yml b/pvc_cp_setup.yml index 4aee30c..e918791 100644 --- a/pvc_cp_setup.yml +++ b/pvc_cp_setup.yml @@ -27,6 +27,12 @@ name: cloudera.exe.init_deployment public: yes when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml tags: - always @@ -34,7 +40,8 @@ hosts: all gather_facts: no tasks: - - ansible.builtin.wait_for_connection: + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: tags: - always @@ -98,6 +105,7 @@ name: cloudera.cluster.config.services.ranger_pvc_default_policies + # TODO: May need to include role to setup dnsmasq for ECS test clusters. prereqs/pvc_ecs & pvc_ecs_dns # ENDBLOCK # PVC Setup # End run From 1b4b03adfeedb999162ce0a84e29db752d76a489 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Thu, 28 Jul 2022 18:58:28 +0100 Subject: [PATCH 4/5] Break pvc playbooks out into groups according to functional area to allow independent execution remove separate pvc-ds playbook as tags and switches better control functionality structured within the main deployment process Signed-off-by: Daniel Chaffelson --- main.yml | 14 +- pvc_cp_setup.yml => pvc_base_postfix.yml | 103 ++++- pvc_base_prereqs_ext.yml | 302 ++++++++++++ pvc_base_prereqs_int.yml | 247 ++++++++++ pvc_base_setup.yml | 558 +---------------------- 5 files changed, 662 insertions(+), 562 deletions(-) rename pvc_cp_setup.yml => pvc_base_postfix.yml (59%) create mode 100644 pvc_base_prereqs_ext.yml create mode 100644 pvc_base_prereqs_int.yml diff --git a/main.yml b/main.yml index 61b1c33..979499d 100644 --- a/main.yml +++ b/main.yml @@ -72,14 +72,24 @@ - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - hostvars.localhost.init__call_cdp_pbc == True +- ansible.builtin.import_playbook: pvc_base_prereqs_ext.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True + +- ansible.builtin.import_playbook: pvc_base_prereqs_int.yml + when: + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime + - hostvars.localhost.init__call_cdp_pvc == True + - ansible.builtin.import_playbook: pvc_base_setup.yml when: - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - hostvars.localhost.init__call_cdp_pvc == True -- ansible.builtin.import_playbook: pvc_cp_setup.yml +- ansible.builtin.import_playbook: pvc_base_postfix.yml when: - - hostvars.localhost.run_pvc + - hostvars.localhost.run_platform or hostvars.localhost.run_runtime - hostvars.localhost.init__call_cdp_pvc == True - ansible.builtin.import_playbook: "{{ [definition_path, 'post_setup.yml'] | path_join }}" diff --git a/pvc_cp_setup.yml b/pvc_base_postfix.yml similarity index 59% rename from pvc_cp_setup.yml rename to pvc_base_postfix.yml index e918791..a1f4e3e 100644 --- a/pvc_cp_setup.yml +++ b/pvc_base_postfix.yml @@ -19,7 +19,7 @@ # This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml # STARTBLOCK # Init run -- name: Init PvC Base Deployment Tasks +- name: Init run tasks for Ansible Controller hosts: localhost gather_facts: yes tasks: @@ -36,17 +36,104 @@ tags: - always -- name: Check Inventory Connectivity +- name: Init run tasks for all nodes hosts: all gather_facts: no tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + - name: Check connectivity to Inventory ansible.builtin.wait_for_connection: tags: - always - # ENDBLOCK # Init run - # STARTBLOCK # PVC Setup +# STARTBLOCK # Fix Auto-TLS + +- name: Auto-TLS Services Setup + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.cms_tls + when: autotls is defined and autotls == True + tags: + - autotls + - never + +# ENDBLOCK # Fix Auto-TLS +# STARTBLOCK # Setup HDFS Encryption + +- name: Setup KTS HA + hosts: localhost + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.deployment.services.kts_high_availability + when: + - "'kts_active' in groups" + - "'kts_passive' in groups" + tags: + - kts + - full_cluster + +- name: Handle KMS services + hosts: localhost + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.deployment.services.kms + when: "'kms_servers' in groups" + - role: cloudera.cluster.deployment.services.kms_ha + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Handle KMS services + hosts: cloudera_manager + gather_facts: no + become: no + roles: + - role: cloudera.cluster.operations.refresh_ranger_kms_repo + when: "'kms_servers' in groups" + tags: + - kms + - full_cluster + +- name: Restart and re-deploy stale client configs + hosts: localhost + gather_facts: no + roles: + - role: cloudera.cluster.operations.restart_stale + when: "'kms_servers' in groups" + vars: + client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" + tags: + - kms + - restart_stale + - full_cluster + + # ENDBLOCK # Setup HDFS Encryption + # STARTBLOCK # WXM Setup + +- name: Handle WXM Setup + hosts: "{{ tp_host | default('cluster_master_nodes[0]') }}" + gather_facts: yes + tags: + - wxm + - full_cluster + tasks: + - name: Setup WXM + when: + - use_wxm | default(False) + - altus_key_id | length > 0 + - altus_private_key | length > 0 + import_role: + name: cloudera.cluster.deployment.services.wxm + +# ENDBLOCK # WXM Setup + - name: Post-Install for PvC on all cluster hosts hosts: cloudera_manager, cluster, ecs_nodes gather_facts: yes @@ -68,6 +155,8 @@ include_role: name: cloudera.cluster.cloudera_manager.services_info public: yes + vars: + cluster_name: "{{ cluster_name_base }}" - name: Fix Hue ticket lifetime for Free IPA include_role: @@ -104,9 +193,5 @@ include_role: name: cloudera.cluster.config.services.ranger_pvc_default_policies - - # TODO: May need to include role to setup dnsmasq for ECS test clusters. prereqs/pvc_ecs & pvc_ecs_dns -# ENDBLOCK # PVC Setup - # End run -### +### \ No newline at end of file diff --git a/pvc_base_prereqs_ext.yml b/pvc_base_prereqs_ext.yml new file mode 100644 index 0000000..b41284a --- /dev/null +++ b/pvc_base_prereqs_ext.yml @@ -0,0 +1,302 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + +# STARTBLOCK # Verify Inventory and Definition +- name: Verify inventory [verify_inventory] + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.verify.inventory + tags: + - verify + - verify_inventory + - default_cluster + - full_cluster + +- name: Verify definition [verify_definition] + hosts: cloudera_manager + gather_facts: no + roles: + - cloudera.cluster.verify.definition + tags: + - verify + - verify_definition + - default_cluster + - full_cluster + +# Moved before parcel verification to allow rehosting +- name: Install custom parcel repository + hosts: custom_repo + become: yes + roles: + - cloudera.cluster.infrastructure.custom_repo + tags: + - custom_repo + - default_cluster + - full_cluster + +# Moved from verify_parcels to reduce duplication +- name: Verify definition [verify_parcels_and_roles] + hosts: cloudera_manager + gather_facts: no + roles: + - cloudera.cluster.verify.parcels_and_roles + tags: + - verify + - verify_parcels + - default_cluster + - full_cluster + +# ENDBLOCK # Verify Inventory and Definition +# STARTBLOCK # Prepare Nodes + +- name: Apply OS pre-requisite configurations + hosts: cloudera_manager, cluster, ca_server, ecs_nodes + become: yes + roles: + - cloudera.cluster.prereqs.os + tags: + - os + - default_cluster + - full_cluster + +- name: Apply OS Prereqs to ECS Nodes + hosts: ecs_nodes + gather_facts: yes + become: yes + tags: + - pvc + - os + - default_cluster + - full_cluster + tasks: + - name: Setup OS Prereqs for ECS Nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.pvc_ecs + + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + +- name: Create local user accounts + hosts: cloudera_manager, cluster + become: yes + gather_facts: no + roles: + - cloudera.cluster.prereqs.user_accounts + tags: + - users + - default_cluster + - full_cluster + +- name: Create local users on ECS Nodes + hosts: ecs_nodes + gather_facts: yes + tags: + - pvc + - users + - default_cluster + - full_cluster + tasks: + - name: Create user accounts for ECS nodes + ansible.builtin.include_role: + name: cloudera.cluster.prereqs.user_accounts_ecs + +# TODO: add pvc tag to all ecs_nodes plays + +- name: Install JDK + hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes + become: yes + roles: + - cloudera.cluster.prereqs.jdk + tags: + - jdk + - security + - free_ipa + - kerberos + - tls + - default_cluster + - full_cluster + +# DB Connectors +- name: Install MySQL Connector + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.mysql_connector + when: database_type == 'mysql' or database_type == 'mariadb' + tags: + - mysql_connector + - full_cluster + +- name: Install Oracle Connector + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: no + become: yes + roles: + - role: cloudera.cluster.prereqs.oracle_connector + when: database_type == 'oracle' + tags: + - oracle_connector + - full_cluster + +# ENDBLOCK # Prepare Nodes +# STARTBLOCK # Install Cluster Service Infrastructure II + +- name: Install RDBMS + hosts: db_server + become: yes + roles: + - cloudera.cluster.infrastructure.rdbms + tags: + - database + - default_cluster + - full_cluster + +# ENDBLOCK # Install Cluster Service Infrastructure II +# STARTBLOCK # Create Cluster Service Infrastructure + +- name: Install Kerberos Server + hosts: krb5_server + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_server + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Setup KRB5 clients + hosts: cloudera_manager, cluster, ecs_nodes + gather_facts: yes + become: yes + roles: + - role: cloudera.cluster.infrastructure.krb5_client + when: "'krb5_server' in groups" + tags: + - security + - kerberos + - free_ipa + - tls + - full_cluster + +- name: Install CA server + hosts: ca_server + become: yes + roles: + - cloudera.cluster.infrastructure.ca_server + tags: + - security + - tls + - full_cluster + +- name: Install HAProxy + hosts: haproxy + become: yes + roles: + - cloudera.cluster.infrastructure.haproxy + tags: + - ha + - full_cluster + +# ENDBLOCK # Create Cluster Service Infrastructure +# STARTBLOCK # Prepare TLS + +- name: Build TLS keystores and truststores + hosts: tls + become: yes + gather_facts: no + roles: + - cloudera.cluster.security.tls_generate_csr + - cloudera.cluster.security.tls_signing + - cloudera.cluster.security.tls_install_certs + vars: + local_certs_dir: "{{ local_temp_dir }}/certs" + local_csrs_dir: "{{ local_temp_dir }}/csrs" + tags: + - tls + - full_cluster + +- name: Delete temp directories + hosts: localhost + become: no + gather_facts: no + tasks: + - file: + path: "{{ [local_temp_dir, dir] | path_join }}" + state: absent + loop: + - csrs + - certs + loop_control: + loop_var: dir + tags: + - always + +# ENDBLOCK # Prepare TLS +# STARTBLOCK # NiFi TLS + +- name: Setup symlinks for NiFi TLS keystore and truststore + hosts: cluster + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.security.tls_nifi + when: > + (tls | default(False) + or manual_tls_cert_distribution | default(False)) + and not (autotls | default(False)) + +# ENDBLOCK # NiFi TLS diff --git a/pvc_base_prereqs_int.yml b/pvc_base_prereqs_int.yml new file mode 100644 index 0000000..01833bc --- /dev/null +++ b/pvc_base_prereqs_int.yml @@ -0,0 +1,247 @@ +--- + +# Copyright 2022 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Playbook is specifically for Deploying Cloudera Clusters +# Edit with extreme caution + +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + +# STARTBLOCK # Install Cloudera Manager + +- name: Install Cloudera Manager daemons + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.daemons + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager server + hosts: cloudera_manager + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.server + tags: + - cm + - default_cluster + - full_cluster + +- name: Install Cloudera Manager License + hosts: cloudera_manager + become: yes + roles: + - role: cloudera.cluster.cloudera_manager.license + tags: + - cm + - license + - default_cluster + - full_cluster + +- name: Install Cloudera Manager agents + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - role: cloudera.cluster.cloudera_manager.agent + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server for TLS + hosts: cloudera_manager + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.server_tls + when: tls | default(False) or manual_tls_cert_distribution | default(False) + tags: + - tls + - cm + - full_cluster + +- name: Configure Cloudera Manager agents + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + any_errors_fatal: true + roles: + - cloudera.cluster.cloudera_manager.agent_config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager server + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.config + tags: + - cm + - default_cluster + - full_cluster + +- name: Configure Cloudera Manager auth and accounts + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.external_auth + - role: cloudera.cluster.cloudera_manager.external_account + tags: + - cm + - full_cluster + +# ENDBLOCK # Install Cloudera Manager +# STARTBLOCK # Cloudera Manager Password + +- name: Configure Cloudera Manager Password + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.admin_password.set + tags: + - cm + - default_cluster + - full_cluster + +- name: Check Cloudera Manager admin password + hosts: cloudera_manager, cluster + gather_facts: no + roles: + - cloudera.cluster.cloudera_manager.api_client + tags: + - cm + - default_cluster + - full_cluster + +# ENDBLOCK # Cloudera Manager Password +# STARTBLOCK # Prepare Security + +- name: Enable Auto-TLS + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.autotls + when: autotls is defined and autotls == True + tags: + - autotls + - full_cluster + +- name: Install pre-requisite packages for Kerberos + hosts: cloudera_manager, cluster, ecs_nodes + become: yes + roles: + - role: cloudera.cluster.prereqs.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - prereqs + - full_cluster + +- name: Configure Cloudera Manager server for Kerberos + hosts: cloudera_manager + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.kerberos + when: krb5_kdc_host is defined or 'krb5_server' in groups + tags: + - kerberos + - full_cluster + +# ENDBLOCK # Prepare Security +# STARTBLOCK # Configure CM + +- name: Restart Cloudera Manager Agents + hosts: cloudera_manager, cluster + gather_facts: no + become: yes + tasks: + - name: Restart Cloudera Manager Agents + meta: noop + notify: + - restart cloudera-scm-agent + tags: + - never + - restart_agents + +- name: Ensure that the agents are heartbeating + hosts: cloudera_manager, cluster + gather_facts: yes + any_errors_fatal: yes + roles: + - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat + when: cloudera_manager_agent_wait_for_heartbeat | default(True) + tags: + - heartbeat + - default_cluster + - full_cluster + +- name: Deploy Cloudera Management Service + hosts: localhost + gather_facts: no + roles: + - cloudera.cluster.deployment.services.mgmt + tags: + - mgmt + - default_cluster + - full_cluster + +- name: Preload parcels from custom repo to Cloudera Manager + hosts: cloudera_manager + become: yes + gather_facts: no + roles: + - role: cloudera.cluster.cloudera_manager.preload_parcels + when: "'custom_repo' in groups" + tags: + - preload_parcels + - default_cluster + - full_cluster + +# ENDBLOCK # Configure CM diff --git a/pvc_base_setup.yml b/pvc_base_setup.yml index ca94828..6ad4463 100644 --- a/pvc_base_setup.yml +++ b/pvc_base_setup.yml @@ -19,7 +19,7 @@ # This plays are imported from a separate playbook so that Ansible tags are intuitively propagated from main.yml # STARTBLOCK # Init run -- name: Init PvC Base Deployment Tasks +- name: Init run tasks for Ansible Controller hosts: localhost gather_facts: yes tasks: @@ -36,468 +36,19 @@ tags: - always -- name: Check Inventory Connectivity +- name: Init run tasks for all nodes hosts: all gather_facts: no tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + - name: Check connectivity to Inventory ansible.builtin.wait_for_connection: tags: - always - # ENDBLOCK # Init run -# STARTBLOCK # Verify Inventory and Definition -# Moved from tasks/group_hosts and tasks/group_hosts_by_tls -- name: Group hosts using host template information - hosts: all - gather_facts: no - roles: - - cloudera.cluster.deployment.groupby - tags: - - always - -- name: Verify inventory [verify_inventory] - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.verify.inventory - tags: - - verify - - verify_inventory - - default_cluster - - full_cluster - -- name: Verify definition [verify_definition] - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.verify.definition - tags: - - verify - - verify_definition - - default_cluster - - full_cluster - -# Moved before parcel verification to allow rehosting -- name: Install custom parcel repository - hosts: custom_repo - become: yes - roles: - - cloudera.cluster.infrastructure.custom_repo - tags: - - custom_repo - - default_cluster - - full_cluster - -# Moved from verify_parcels to reduce duplication -- name: Verify definition [verify_parcels_and_roles] - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera.cluster.verify.parcels_and_roles - tags: - - verify - - verify_parcels - - default_cluster - - full_cluster - -# ENDBLOCK # Verify Inventory and Definition -# STARTBLOCK # Prepare Nodes - -- name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster, ca_server, ecs_nodes - become: yes - roles: - - cloudera.cluster.prereqs.os - tags: - - os - - default_cluster - - full_cluster - -- name: Apply OS Prereqs to ECS Nodes - hosts: ecs_nodes - gather_facts: yes - become: yes - tags: - - pvc - - os - - default_cluster - - full_cluster - tasks: - - name: Setup iptables or nftables - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.pvc_ecs - - - name: Create user accounts for ECS nodes - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.user_accounts_ecs - -- name: Create local user accounts - hosts: cloudera_manager, cluster - become: yes - gather_facts: no - roles: - - cloudera.cluster.prereqs.user_accounts - tags: - - users - - default_cluster - - full_cluster - -- name: Create local users on ECS Nodes - hosts: ecs_nodes - gather_facts: yes - tags: - - pvc - - users - - default_cluster - - full_cluster - tasks: - - name: Create user accounts for ECS nodes - ansible.builtin.include_role: - name: cloudera.cluster.prereqs.user_accounts_ecs - -- name: Install JDK - hosts: cloudera_manager, cluster, tls, krb5_server, ecs_nodes - become: yes - roles: - - cloudera.cluster.prereqs.jdk - tags: - - jdk - - security - - free_ipa - - kerberos - - tls - - default_cluster - - full_cluster - -# DB Connectors -- name: Install MySQL Connector - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.mysql_connector - when: database_type == 'mysql' or database_type == 'mariadb' - tags: - - mysql_connector - - full_cluster - -- name: Install Oracle Connector - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.prereqs.oracle_connector - when: database_type == 'oracle' - tags: - - oracle_connector - - full_cluster - -# ENDBLOCK # Prepare Nodes -# STARTBLOCK # Install Cluster Service Infrastructure II - -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - cloudera.cluster.infrastructure.rdbms - tags: - - database - - default_cluster - - full_cluster - -# ENDBLOCK # Install Cluster Service Infrastructure II -# STARTBLOCK # Create Cluster Service Infrastructure - -- name: Install Kerberos Server - hosts: krb5_server - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_server - tags: - - security - - kerberos - - free_ipa - - tls - - full_cluster - -- name: Setup KRB5 clients - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - roles: - - role: cloudera.cluster.infrastructure.krb5_client - when: "'krb5_server' in groups" - tags: - - security - - kerberos - - free_ipa - - tls - - full_cluster - -- name: Install CA server - hosts: ca_server - become: yes - roles: - - cloudera.cluster.infrastructure.ca_server - tags: - - security - - tls - - full_cluster - -- name: Install HAProxy - hosts: haproxy - become: yes - roles: - - cloudera.cluster.infrastructure.haproxy - tags: - - ha - - full_cluster - -# ENDBLOCK # Create Cluster Service Infrastructure -# STARTBLOCK # Prepare TLS - -- name: Build TLS keystores and truststores - hosts: tls - become: yes - gather_facts: no - roles: - - cloudera.cluster.security.tls_generate_csr - - cloudera.cluster.security.tls_signing - - cloudera.cluster.security.tls_install_certs - vars: - local_certs_dir: "{{ local_temp_dir }}/certs" - local_csrs_dir: "{{ local_temp_dir }}/csrs" - tags: - - tls - - full_cluster - -- name: Delete temp directories - hosts: localhost - become: no - gather_facts: no - tasks: - - file: - path: "{{ [local_temp_dir, dir] | path_join }}" - state: absent - loop: - - csrs - - certs - loop_control: - loop_var: dir - tags: - - always - -# ENDBLOCK # Prepare TLS -# STARTBLOCK # NiFi TLS - -- name: Setup symlinks for NiFi TLS keystore and truststore - hosts: cluster - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.security.tls_nifi - when: > - (tls | default(False) - or manual_tls_cert_distribution | default(False)) - and not (autotls | default(False)) - -# ENDBLOCK # NiFi TLS -# STARTBLOCK # Install Cloudera Manager - -- name: Install Cloudera Manager daemons - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.daemons - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager server - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.server - tags: - - cm - - default_cluster - - full_cluster - -- name: Install Cloudera Manager License - hosts: cloudera_manager - become: yes - roles: - - role: cloudera.cluster.cloudera_manager.license - tags: - - cm - - license - - default_cluster - - full_cluster - -- name: Install Cloudera Manager agents - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - role: cloudera.cluster.cloudera_manager.agent - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server for TLS - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.server_tls - when: tls | default(False) or manual_tls_cert_distribution | default(False) - tags: - - tls - - cm - - full_cluster - -- name: Configure Cloudera Manager agents - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - any_errors_fatal: true - roles: - - cloudera.cluster.cloudera_manager.agent_config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager server - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.config - tags: - - cm - - default_cluster - - full_cluster - -- name: Configure Cloudera Manager auth and accounts - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.external_auth - - role: cloudera.cluster.cloudera_manager.external_account - tags: - - cm - - full_cluster - -# ENDBLOCK # Install Cloudera Manager -# STARTBLOCK # Cloudera Manager Password - -- name: Configure Cloudera Manager Password - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.admin_password.set - tags: - - cm - - default_cluster - - full_cluster - -- name: Check Cloudera Manager admin password - hosts: cloudera_manager, cluster - gather_facts: no - roles: - - cloudera.cluster.cloudera_manager.api_client - tags: - - cm - - default_cluster - - full_cluster - -# ENDBLOCK # Cloudera Manager Password -# STARTBLOCK # Prepare Security - -- name: Enable Auto-TLS - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.autotls - when: autotls is defined and autotls == True - tags: - - autotls - - full_cluster - -- name: Install pre-requisite packages for Kerberos - hosts: cloudera_manager, cluster, ecs_nodes - become: yes - roles: - - role: cloudera.cluster.prereqs.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - prereqs - - full_cluster - -- name: Configure Cloudera Manager server for Kerberos - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - full_cluster - -# ENDBLOCK # Prepare Security -# STARTBLOCK # Configure CM - -- name: Restart Cloudera Manager Agents - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - tasks: - - name: Restart Cloudera Manager Agents - meta: noop - notify: - - restart cloudera-scm-agent - tags: - - never - - restart_agents - -- name: Ensure that the agents are heartbeating - hosts: cloudera_manager, cluster - gather_facts: yes - any_errors_fatal: yes - roles: - - role: cloudera.cluster.cloudera_manager.wait_for_heartbeat - when: cloudera_manager_agent_wait_for_heartbeat | default(True) - tags: - - heartbeat - - default_cluster - - full_cluster - -- name: Deploy Cloudera Management Service - hosts: localhost - gather_facts: no - roles: - - cloudera.cluster.deployment.services.mgmt - tags: - - mgmt - - default_cluster - - full_cluster - -- name: Preload parcels from custom repo to Cloudera Manager - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.preload_parcels - when: "'custom_repo' in groups" - tags: - - preload_parcels - - default_cluster - - full_cluster - -# ENDBLOCK # Configure CM # STARTBLOCK # Install Cluster - name: Deploy clusters @@ -511,101 +62,6 @@ - full_cluster # ENDBLOCK # Install Cluster -# STARTBLOCK # Fix Auto-TLS - -- name: Auto-TLS Services Setup - hosts: cloudera_manager - gather_facts: no - roles: - - role: cloudera.cluster.cloudera_manager.cms_tls - when: autotls is defined and autotls == True - tags: - - autotls - - never - -# ENDBLOCK # Fix Auto-TLS -# STARTBLOCK # Setup HDFS Encryption - -- name: Setup KTS HA - hosts: localhost - become: yes - gather_facts: no - roles: - - role: cloudera.cluster.deployment.services.kts_high_availability - when: - - "'kts_active' in groups" - - "'kts_passive' in groups" - tags: - - kts - - full_cluster - -- name: Handle KMS services - hosts: localhost - gather_facts: no - become: yes - roles: - - role: cloudera.cluster.deployment.services.kms - when: "'kms_servers' in groups" - - role: cloudera.cluster.deployment.services.kms_ha - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Handle KMS services - hosts: cloudera_manager - gather_facts: no - become: no - roles: - - role: cloudera.cluster.operations.refresh_ranger_kms_repo - when: "'kms_servers' in groups" - tags: - - kms - - full_cluster - -- name: Restart and re-deploy stale client configs - hosts: localhost - gather_facts: no - roles: - - role: cloudera.cluster.operations.restart_stale - when: "'kms_servers' in groups" - vars: - client_config_timeout: "{{ restart_client_config_timeout | default(300) }}" - tags: - - kms - - restart_stale - - full_cluster - - # ENDBLOCK # Setup HDFS Encryption - # STARTBLOCK # PVC Setup - -- name: Prepare Hosts for PvC Setup - hosts: cloudera_manager, cluster, ecs_nodes - gather_facts: yes - tags: - - pvc - tasks: - - name: Add missing ExtJS for Oozie UI - include_role: - name: cloudera.cluster.config.services.oozie_ui - when: oozie_service_exists | default(false) - -- name: Handle WXM Setup - hosts: "{{ tp_host | default('cluster_master_nodes[0]') }}" - gather_facts: yes - tags: - - wxm - - full_cluster - tasks: - - name: Setup WXM - when: - - use_wxm | default(False) - - altus_key_id | length > 0 - - altus_private_key | length > 0 - import_role: - name: cloudera.cluster.deployment.services.wxm - -# ENDBLOCK # PVC Setup # End run -### +### \ No newline at end of file From 4e69ed2970b3a4514f86ed8463890c9730815239 Mon Sep 17 00:00:00 2001 From: Daniel Chaffelson Date: Sun, 31 Jul 2022 10:23:44 +0100 Subject: [PATCH 5/5] Add Teardown play for ECS clusters Signed-off-by: Daniel Chaffelson --- pvc_base_teardown.yml | 47 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/pvc_base_teardown.yml b/pvc_base_teardown.yml index 4528c8c..e79826c 100644 --- a/pvc_base_teardown.yml +++ b/pvc_base_teardown.yml @@ -14,6 +14,38 @@ # See the License for the specific language governing permissions and # limitations under the License. +# STARTBLOCK # Init run +- name: Init run tasks for Ansible Controller + hosts: localhost + gather_facts: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always +# ENDBLOCK # Init run + # STARTBLOCK # Teardown # Teardown CA @@ -40,7 +72,20 @@ # Teardown Cluster -- name: Teardown +- name: Teardown ECS + hosts: ecs_nodes + gather_facts: no + become: yes + tasks: + - ansible.builtin.include_role: + name: cloudera.cluster.teardown + tasks_from: teardown_ecs.yml + vars: + cluster: "{{ definition.clusters | selectattr('type', 'defined') | selectattr('type', 'search', 'ecs') | first }}" + tags: + - teardown_ecs + +- name: Cluster Teardown Process hosts: all gather_facts: no become: yes