diff --git a/main.yml b/main.yml index bbb2489..cc286e9 100644 --- a/main.yml +++ b/main.yml @@ -28,7 +28,7 @@ - name: Import the Cloud Interactions Playbook import_playbook: cloud.yml -- name: Teardown Dynamic Inventory when requested +- name: Teardown Cleanup hosts: localhost tags: [teardown,never] gather_facts: yes @@ -38,6 +38,15 @@ name: cloudera_deploy tasks_from: clean_dynamic_inventory + - name: Remove Terraform remote state resources if requested + when: + - globals.infra_deployment_engine == 'terraform' + - globals.terraform_auto_remote_state | bool + - globals.terraform_state_storage in ['remote_s3'] + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: auto_terraform_state + - name: Prepare for Cloudera Cluster Run hosts: localhost tags: always diff --git a/readme.adoc b/readme.adoc index 9cac8e4..a32d350 100644 --- a/readme.adoc +++ b/readme.adoc @@ -447,6 +447,46 @@ WARNING: Setting a deployment to a lower runlevel, e.g. from `run` to `infra` wi For further details on the various _runlevel_-like tags for CDP Public Cloud, see the https://github.com/cloudera-labs/cloudera.exe/blob/main/docs/runlevels.md[Runlevel Guide] in the `cloudera.exe` project. +=== Terraform Deployment Engine + +Terraform can optionally be used to create the cloud infrastructure. This will attempt to create the cloud provider assets at the `infra` (network, storage and compute) and `plat` (IAM policies and roles) runlevels using Terraform resources. A list of Terraform related parameters are shown in the table below. + +.List of parameters used by Terraform deployment engine +[cols="1,1,1,1"] +|=== +|Parameter|Description|Default Value|Notes + +|`infra_deployment_engine` +|The engine (ansible or terraform) that will be used to create the infrastructure resources. +| `ansible` +| Needs to be set to `terraform` for Terraform-deployment. + +|`terraform_base_dir` +| Top-level directory where all Terraform assets will be placed. Includes processed Jinja template files for Terraform, timestamped artefact of Terraform files and the workspace directory where terraform apply/destroy is run. +| `~/.config/cloudera-deploy/terraform` +| + +|`terraform_state_storage` +|The type of backend storage to use for the Terraform state. +| `local` +| Current options are `local` or `remote_s3` + +|`terraform_auto_remote_state` +| Flag to allow Cloudera Deploy automatically provision remote state resources as part of its initialization. This will also teardown these resources during cleanup. +| `False` +| + +|`terraform_remote_state_bucket` +|The name of the Terraform state storage bucket. +| +| Required if using `remote_s3` state storage. Value is derived from `name_prefix` if terraform_auto_remote_state is True. + +|`terraform_remote_state_lock_table` +|The name of the table to track locks of remote Terraform state. +| +| Required if using `remote_s3` state storage. Value is derived from `name_prefix` if terraform_auto_remote_state is True. +|=== + == Definitions Cloudera Deploy uses a set of configuration files within a directory to define and coordinate a deployment. This directory also stores any artifacts created during the deployment, such as Ansible inventory files, CDP environment readouts, etc. diff --git a/roles/cloudera_deploy/defaults/main.yml b/roles/cloudera_deploy/defaults/main.yml index 35e16e0..bcdebbe 100644 --- a/roles/cloudera_deploy/defaults/main.yml +++ b/roles/cloudera_deploy/defaults/main.yml @@ -47,5 +47,9 @@ default_parcel_distro: el7 # el8, bionic, focal default_download_link_expiry: 3600 # Default Deployment Controls +default_infra_deployment_engine: ansible default_infra_type: aws default_infra_region: us-east-1 + +# Terraform defaults +default_terraform_base_dir: "{{ [default_config_path, 'terraform'] | path_join }}" \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/auto_terraform_state.yml b/roles/cloudera_deploy/tasks/auto_terraform_state.yml new file mode 100644 index 0000000..82bf2ed --- /dev/null +++ b/roles/cloudera_deploy/tasks/auto_terraform_state.yml @@ -0,0 +1,48 @@ +--- + +# Copyright 2021 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Resources for remote_s3 state storage + when: + - globals.terraform_state_storage == 'remote_s3' + block: + + # Create or Teardown the resources + - name: AWS Bucket for Remote State Storage + amazon.aws.aws_s3: + region: "{{ globals.region }}" + bucket: "{{ globals.terraform_remote_state_bucket}}" + mode: "{{ ('teardown' not in ansible_run_tags) | ternary('create', 'delete') }}" # Check ansible tag to determine action + permission: private + register: __infra_aws_storage_locations_info + + - name: AWS DynamoDB for Remote State Locking + community.aws.dynamodb_table: + region: "{{ globals.region }}" + name: "{{ globals.terraform_remote_state_lock_table }}" + read_capacity: 1 + write_capacity: 1 + hash_key_name: LockID + hash_key_type: STRING + state: "{{ ('teardown' not in ansible_run_tags) | ternary('present', 'absent') }}" # Check ansible tag to determine action + + - name: Print remote state configuration + when: "'teardown' not in ansible_run_tags" + ansible.builtin.debug: + msg: + - "Resources for remote_s3 Terraform State created." + - "S3 Bucket Name: {{ globals.terraform_remote_state_bucket}}" + - "DynamoDB Locking Table: {{ globals.terraform_remote_state_lock_table}}" + verbosity: 3 \ No newline at end of file diff --git a/roles/cloudera_deploy/tasks/init.yml b/roles/cloudera_deploy/tasks/init.yml index fec411c..65670bc 100644 --- a/roles/cloudera_deploy/tasks/init.yml +++ b/roles/cloudera_deploy/tasks/init.yml @@ -173,7 +173,13 @@ name_prefix: "{{ name_prefix | default(default_name_prefix) }}" tags: "{{ tags | default(omit) }}" region: "{{ infra_region | default(default_infra_region) }}" + infra_deployment_engine: "{{ infra_deployment_engine | default(default_infra_deployment_engine) }}" infra_type: "{{ infra_type | default(default_infra_type) }}" + terraform_base_dir: "{{ terraform_base_dir | default(default_terraform_base_dir) | expanduser }}" + terraform_state_storage: "{{ terraform_state_storage | default(omit) }}" + terraform_auto_remote_state: "{{ terraform_auto_remote_state | default(False) }}" + terraform_remote_state_bucket: "{{ terraform_remote_state_bucket | default(omit) }}" + terraform_remote_state_lock_table: "{{ terraform_remote_state_lock_table | default(omit) }}" ssh: public_key_id: "{{ public_key_id | default(omit) }}" public_key_file: "{{ public_key_file | default(omit) }}" @@ -217,6 +223,31 @@ fail_msg: "You must supply a valid Namespace" quiet: yes +- name: Check Deployment Engine variable + ansible.builtin.assert: + that: + - globals.infra_deployment_engine in ['ansible', 'terraform'] + fail_msg: "The 'infra_deployment_engine' variable must be one of 'ansible', 'terraform'" + +- name: Check Supplied terraform_base_dir variable + when: + - infra_deployment_engine == 'terraform' + ansible.builtin.assert: + that: + - globals.terraform_base_dir is defined + - globals.terraform_base_dir | length > 0 + fail_msg: "You must supply a 'terraform_base_dir' where Terraform assets will be placed" + quiet: yes + +- name: Check Supplied terraform_auto_remote_state variable + when: + - infra_deployment_engine == 'terraform' + ansible.builtin.assert: + that: + - (globals.terraform_auto_remote_state|bool is sameas true) or (globals.terraform_auto_remote_state|bool is sameas false) + fail_msg: "The terraform_auto_remote_state variable must be a boolean variable" + quiet: yes + # SSH - name: Use default SSH public key id when: globals.ssh.public_key_id is undefined @@ -405,3 +436,44 @@ fail_msg: >- Admin Password must comply with CDP Public requirements: 1 Upper, 1 Special, 1 Number, 8-64 chars. quiet: yes + +# Provision Terraform remote state resources if requested +- name: Initialize Terraform remote state resources + when: + - init__call_cloud_role | bool + - globals.infra_deployment_engine == 'terraform' + - globals.terraform_auto_remote_state | bool + - globals.terraform_state_storage in ['remote_s3'] + block: + # Set resource variable names if not already done + - name: Set variables for remote state bucket if not set + when: (globals.terraform_remote_state_bucket is not defined) or + ( (globals.terraform_remote_state_bucket) | length == 0) + ansible.builtin.set_fact: + globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" + vars: + remote_state_vars: + terraform_remote_state_bucket: "{{ [globals.name_prefix, 'state-bucket'] | join('-') }}" + + - name: Set variables for remote state lock table if not set + when: (globals.terraform_remote_state_lock_table is not defined) or + (globals.terraform_remote_state_lock_table | length == 0) + ansible.builtin.set_fact: + globals: "{{ globals | default({}) | combine(remote_state_vars, recursive=True) }}" + vars: + remote_state_vars: + terraform_remote_state_lock_table: "{{ [globals.name_prefix, 'state-lock-table'] | join('-') }}" + + - name: Create remote state resources + when: "'teardown' not in ansible_run_tags" + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: auto_terraform_state + +- name: Prepare for Download Mirror Population if requested and necessary + when: + - use_download_mirror | default(default_enable_download_mirror) | bool + - "'teardown' not in ansible_run_tags" + ansible.builtin.include_role: + name: cloudera_deploy + tasks_from: prepare_download_mirror \ No newline at end of file