Add Terraform deployment engine for cloud resources

roles/common/defaults/main.yml

@@ -57,6 +57,7 @@ common__ngw_suffix:                       "{{ globals.labels.nat_gateway | defau
 common__unique_storage_name_suffix:       "{{ globals.storage.name | default((common__region + common__aws_profile) if 'aws' in common__infra_type else common__region) }}"
 
 # Infra
+common__infra_deployment_engine:          "{{ globals.infra_deployment_engine | default('ansible') }}"
 common__infra_type:                       "{{ globals.infra_type | default('aws') }}"
 common__public_key_file:                  "{{ globals.ssh.public_key_file | default('') }}"
 common__namespace_cdp:                    "{{ globals.namespace_cdp | default([common__namespace, common__namespace_unique_suffix] | join('-')) }}"
@@ -66,6 +67,20 @@ common__public_key_text:                  "{{ globals.ssh.public_key_text | defa
 common__region:                           "{{ globals.region | default(common__region_default[common__infra_type]) }}"
 common__storage_name:                     "{{ infra.storage.name | default([common__namespace, common__unique_storage_name_suffix[::2] | replace('-','')] | join('-')) }}"
 
+# Terraform
+common__terraform_base_dir:               "{{ globals.terraform_base_dir | default( [playbook_dir , 'terraform'] | path_join ) }}"
+# The processed Jinja template files for Terraform are placed in common__terraform_template_dir
+common__terraform_template_dir:            "{{ [common__terraform_base_dir , 'processed_template_code'] | path_join }}"
+# A timestamped artefact directory storing a copy of the Terraform code from each run 
+common__terraform_artefact_dir:            "{{ [common__terraform_base_dir , ('tf_artefacts_' + ansible_date_time.iso8601 ) ] | path_join | regex_replace(':','_')}}"
+# Terraform apply/destroy run from under this directory
+common__terraform_workspace_dir:           "{{ [common__terraform_base_dir, 'workspace'] | path_join }}"
+
+common__terraform_allowed_state_storage:   "['local', 'remote_s3']"
+common__terraform_state_storage:           "{{ globals.terraform_state_storage  | default('local') }}"
+common__terraform_remote_state_bucket:     "{{ globals.terraform_remote_state_bucket | default('') }}"      
+common__terraform_remote_state_lock_table: "{{ globals.terraform_remote_state_lock_table | default('') }}"      
+
 common__vpc_name:                         "{{ infra.vpc.name | default([common__namespace, common__vpc_name_suffix] | join('-')) }}"
 common__vpc_public_subnet_cidrs:          "{{ infra.vpc.public_subnets | default(['10.10.0.0/19', '10.10.32.0/19', '10.10.64.0/19']) }}"
 common__vpc_private_subnet_cidrs:         "{{ infra.vpc.private_subnets | default(['10.10.96.0/19', '10.10.128.0/19', '10.10.160.0/19']) }}"

roles/infrastructure/defaults/main.yml

@@ -26,6 +26,18 @@ infra__vpc_svcnet_suffix:           "{{ common__vpc_svcnet_suffix }}"
 infra__vpc_private_subnets_suffix:  "{{ common__vpc_private_subnets_suffix }}"
 infra__vpc_public_subnets_suffix:   "{{ common__vpc_public_subnets_suffix }}"
 
+# Deployment type
+infra__deployment_engine:                 "{{ common__infra_deployment_engine }}"
+# Location of output from template module which creates Terraform
+infra__terraform_template_dir:            "{{ common__terraform_template_dir }}"
+infra__terraform_artefact_dir:            "{{ common__terraform_artefact_dir }}"
+infra__terraform_workspace_dir:           "{{ common__terraform_workspace_dir }}"      
+
+infra__terraform_allowed_state_storage:   "{{ common__terraform_allowed_state_storage }}"
+infra__terraform_state_storage:           "{{ common__terraform_state_storage }}"
+infra__terraform_remote_state_bucket:     "{{ common__terraform_remote_state_bucket }}"      
+infra__terraform_remote_state_lock_table: "{{ common__terraform_remote_state_lock_table }}"      
+
 # Infra
 infra__type:                        "{{ common__infra_type }}"
 infra__tunnel:                      "{{ common__tunnel }}"

roles/infrastructure/tasks/initialize_aws_terraform.yml

@@ -0,0 +1,64 @@
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create directory for processed Terraform template files
+  ansible.builtin.file:
+    path: "{{ infra__terraform_template_dir }}/infra"
+    state: directory
+
+- name: Create artefact directory for Terraform infra code
+  ansible.builtin.file:
+    path: "{{ infra__terraform_artefact_dir }}/infra"
+    state: directory
+
+# Apply template for Terraform provider
+- name: Generate Terraform Provider
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/provider.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/provider.tf"
+
+# Apply template for Terraform backend state
+- name: Generate Terraform Backend State
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/backend_state.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf"
+
+# Apply template for Terraform variables
+- name: Generate Terraform Variables
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/terraform_variables.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/variables.tf"
+  no_log: false
+
+# Apply template for Terraform infra....
+# ...network resources
+- name: Generate Terraform infra file for network resources
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/infra_{{ infra__type }}_network.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/infra_network.tf"
+  no_log: false
+
+# ...storage resources
+- name: Generating Terraform infra file for storage resources
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/infra_{{ infra__type }}_storage.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/infra_storage.tf"
+  no_log: false
+
+# ...compute resources
+- name: Generating Terraform infra file for compute resources
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/infra_{{ infra__type }}_compute.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/infra_compute.tf"
+  no_log: false

roles/infrastructure/tasks/initialize_setup.yml

@@ -22,4 +22,8 @@
   ansible.builtin.include_tasks: "initialize_{{ infra__type | lower }}.yml"
 
 - name: Include tasks fpr provider-specific Infrastructure setup initialization
-  ansible.builtin.include_tasks: "initialize_setup_{{ infra__type | lower }}.yml"
+  ansible.builtin.include_tasks: "initialize_setup_{{ infra__type | lower }}.yml"
+
+- name: Include Terraform tasks for provider-specific Infrastructure initialization
+  ansible.builtin.include_tasks: "initialize_{{ infra__type | lower }}_{{ infra__deployment_engine }}.yml"
+  when: infra__deployment_engine == 'terraform'

roles/infrastructure/tasks/initialize_teardown.yml

@@ -21,4 +21,8 @@
   ansible.builtin.include_tasks: "initialize_{{ infra__type | lower }}.yml"
 
 - name: Include tasks for provider-specific Infrastructure teardown initialization
-  ansible.builtin.include_tasks: "initialize_teardown_{{ infra__type | lower }}.yml"
+  ansible.builtin.include_tasks: "initialize_teardown_{{ infra__type | lower }}.yml"
+
+- name: Include Terraform tasks for provider-specific Infrastructure teardown initialization
+  ansible.builtin.include_tasks: "initialize_teardown_{{ infra__type | lower }}_{{ infra__deployment_engine }}.yml"
+  when: infra__deployment_engine == 'terraform'

roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml

@@ -0,0 +1,37 @@
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create directory for processed Terraform template files
+  ansible.builtin.file:
+    path: "{{ infra__terraform_template_dir }}/infra"
+    state: directory
+
+# Apply template for Terraform provider
+- name: Generate Terraform Provider
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/provider.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/provider.tf"
+
+# Apply template for Terraform backend state
+- name: Generate Terraform Backend State
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/backend_state.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf"
+
+# Apply template for Terraform variables
+- name: Generate Terraform Variables
+  ansible.builtin.template:
+    src: 'template/{{ infra__type }}/terraform_variables.tf.j2'
+    dest: "{{ infra__terraform_template_dir }}/infra/variables.tf"
+  no_log: false

roles/infrastructure/tasks/setup.yml

@@ -19,16 +19,25 @@
 - name: Set up provider-specific Infrastructure artifacts
   ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}.yml"
 
-- name: Set up provider-specific Infrastructure network
-  ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_network.yml"
+- name: Set up for Ansible deployment engine
+  when: infra__deployment_engine == 'ansible'
+  block:
+  - name: Set up provider-specific Infrastructure network
+    ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_network.yml"
 
-- name: Set up provider-specific Infrastructure storage
-  ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_storage.yml"
+  - name: Set up provider-specific Infrastructure storage
+    ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_storage.yml"
 
-- name: Set Up localised provider-specific Storage Utility Service
-  when: infra__create_utility_service
-  ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_utility_service.yml"
+  - name: Set Up localised provider-specific Storage Utility Service
+    when: infra__create_utility_service
+    ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_utility_service.yml"
 
-- name: Set up provider-specific Infrastructure Compute
-  when: infra__dynamic_inventory_count
-  ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_compute.yml"
+  - name: Set up provider-specific Infrastructure Compute
+    when: infra__dynamic_inventory_count
+    ansible.builtin.include_tasks: "setup_{{ infra__type | lower }}_compute.yml"
+
+- name: Set up for Terraform deployment engine
+  when: infra__deployment_engine == 'terraform'
+  block:
+  - name: Set up Terraform Infrastructure artifacts
+    ansible.builtin.include_tasks: "setup_{{ infra__deployment_engine }}.yml"

roles/infrastructure/tasks/setup_terraform.yml

@@ -0,0 +1,88 @@
+---
+
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Ensure the Terraform workspace directory exists
+  ansible.builtin.copy:
+    src: "{{ infra__terraform_template_dir }}/infra/"
+    dest: "{{ infra__terraform_workspace_dir }}/infra"
+
+- name: Copy Terraform infra code to the artefact directory
+  ansible.builtin.copy:
+    src: "{{ infra__terraform_template_dir }}/infra/"
+    dest: "{{ infra__terraform_artefact_dir }}/infra"
+
+- name: Applying Terraform
+  community.general.terraform:
+    project_path: "{{ infra__terraform_workspace_dir }}/infra"
+    state: "present"
+    force_init: yes
+  register: tf_result
+  retries: 3
+  delay: 10
+  until: tf_result is succeeded
+
+- name: Remove the Terraform template directory
+  ansible.builtin.file:
+    path: "{{ infra__terraform_template_dir }}/infra"
+    state: absent
+
+- name: Remove the Terraform workspace directory when using remote state
+  ansible.builtin.file:
+    path: "{{ infra__terraform_workspace_dir }}/infra"
+    state: absent
+  when: infra__terraform_state_storage in ['remote_s3']
+
+# Get information about Dynamic Inventory VMs if created via Terraform
+- name: Fetch EC2 Instance info for Dynamic Inventory Nodes
+  register: __infra_dynamic_inventory_discovered
+  community.aws.ec2_instance_info:
+    region: "{{ infra__region }}"
+    filters: "{{ __filters | items2dict }}"
+  vars:
+    __filters:
+      - key: "tag:{{ infra__dynamic_inventory_tag_key }}"
+        value: "{{ infra__dynamic_inventory_tag_value }}*"
+
+- name: Create output Dictionary for producing Static Inventory artefact
+  ansible.builtin.set_fact:
+    infra__dynamic_inventory_host_entries: "{{ infra__dynamic_inventory_host_entries | default([]) | union([host_entry]) }}"
+  vars:
+    host_entry: "{{ [__infra_di_item.private_dns_name, 'ansible_host=' + __infra_di_item.public_ip_address, infra__dynamic_inventory_connectors] | join(' ') }}"
+  loop: "{{ __infra_dynamic_inventory_discovered.instances }}"
+  loop_control:
+    loop_var: __infra_di_item
+
+# If created Utility Instance via Terraform then
+# need to get it's info and add to an Ansible host group
+- name: Add Utility Instance to host group
+  when: infra__create_utility_service
+  block:
+    - name: Discover the Utility Instance details
+      community.aws.ec2_instance_info:
+        region: "{{ infra__region }}"
+        filters: "{{ __filters | items2dict }}"
+      vars:
+        __filters:
+          - key: "tag:Name"
+            value: "{{ infra__namespace }}*"
+      register: __infra_utility_compute_discovered
+
+    - name: Add discovered Utility Instance to host group
+      ansible.builtin.add_host:
+        hostname: "{{__infra_utility_compute_discovered.instances[0].public_ip_address}}"
+        ansible_user: "{{ infra__dynamic_inventory_images_default[infra__type][infra__dynamic_inventory_os].user }}"
+        ansible_ssh_private_key_file: "{{ (infra__private_key_file == '') | ternary(omit, infra__private_key_file) }}"
+        groupname: cldr_utility

roles/infrastructure/tasks/teardown.yml

@@ -14,17 +14,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Teardown provider-specific Infrastructure compute
-  ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_compute.yml"
+- name: Teardown for Ansible deployment engine
+  when: infra__deployment_engine == 'ansible'
+  block:
+  - name: Teardown provider-specific Infrastructure compute
+    ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_compute.yml"
 
-- name: Teardown provider-specific Infrastructure storage
-  ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_storage.yml"
+  - name: Teardown provider-specific Infrastructure storage
+    ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_storage.yml"
 
-- name: Teardown provider-specific Infrastructure network
-  ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_network.yml"
+  - name: Teardown provider-specific Infrastructure network
+    ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_network.yml"
 
-- name: Teardown provider-specific Utility Services
-  ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_utility_service.yml"
+  - name: Teardown provider-specific Utility Services
+    ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}_utility_service.yml"
 
-- name: Teardown provider-specific Infrastructure artifacts
-  ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}.yml"
+  - name: Teardown provider-specific Infrastructure artifacts
+    ansible.builtin.include_tasks: "teardown_{{ infra__type | lower }}.yml"
+
+- name: Teardown for Terraform deployment engine
+  when: infra__deployment_engine == 'terraform'
+  block:
+  - name: Teardown Terraform Infrastructure artifacts
+    ansible.builtin.include_tasks: "teardown_{{ infra__deployment_engine }}.yml"

roles/infrastructure/tasks/teardown_terraform.yml

@@ -0,0 +1,40 @@
+---
+
+# Copyright 2021 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Ensure the Terraform workspace directory exists
+  ansible.builtin.copy:
+    src: "{{ infra__terraform_template_dir }}/infra/"
+    dest: "{{ infra__terraform_workspace_dir }}/infra"
+
+- name: Destroy Terraform infra resources
+  community.general.terraform:
+    project_path: "{{ infra__terraform_workspace_dir }}/infra"
+    state: "absent"
+    force_init: yes
+  register: tf_result
+  retries: 3
+  delay: 10
+  until: tf_result is succeeded
+
+- name: Remove the Terraform template directory
+  ansible.builtin.file:
+    path: "{{ infra__terraform_template_dir }}/infra"
+    state: absent
+
+- name: Remove the Terraform workspace directory
+  ansible.builtin.file:
+    path: "{{ infra__terraform_workspace_dir }}/infra"
+    state: absent

roles/infrastructure/tasks/validate.yml

@@ -22,5 +22,10 @@
       - infra__private_key_file is string
     fail_msg: "Invalid SSH private key file for Dynamic Inventory. Key file must be a string."
 
-- name: Validate provider-specific Infrastructure parameters
+- name: Validate provider-specific Infrastructure parameters - Ansible
   ansible.builtin.include_tasks: "validate_{{ infra__type | lower }}.yml"
+  when: infra__deployment_engine == 'ansible'
+
+- name: Validate Terraform provider-specific Infrastructure parameters
+  ansible.builtin.include_tasks: "validate_{{ infra__type | lower }}_{{ infra__deployment_engine }}.yml"
+  when: infra__deployment_engine == 'terraform'