diff --git a/roles/infrastructure/files/aws/infra_aws_compute.tf b/roles/infrastructure/files/aws/infra_aws_compute.tf new file mode 100644 index 00000000..1fb347c4 --- /dev/null +++ b/roles/infrastructure/files/aws/infra_aws_compute.tf @@ -0,0 +1,52 @@ +# ------- Dynamic Inventory VMs ------- +resource "aws_instance" "cdp_dynamic_inventory_vm" { + + for_each = {for idx, vm in var.dynamic_inventory_vms: idx => vm} + + vpc_security_group_ids = [aws_security_group.cdp_default_sg.id] + key_name = var.dynamic_inventory_public_key_id + instance_type = each.value.instance_type + ami = each.value.ami + ebs_optimized = true + + # Volume / root_block_device + root_block_device { + delete_on_termination = each.value.volume.delete_on_termination + volume_size = each.value.volume.volume_size + volume_type = each.value.volume.volume_type + } + + # TODO: Review settng subnet_id to first public subnet (believe Ansible approach does this) + subnet_id = aws_subnet.cdp_public_subnets[0].id + + associate_public_ip_address = true + + tags = merge(var.dynamic_inventory_tags,{Name = each.value.name}) +} + +# ------- Localised Utility VM Instance ------- +resource "aws_instance" "cdp_utility_vms" { + + for_each = {for idx, vm in var.utility_vms: idx => vm} + + vpc_security_group_ids = [aws_security_group.cdp_default_sg.id] + key_name = var.utility_vm_public_key_id + instance_type = each.value.instance_type + ami = each.value.ami + ebs_optimized = true + + subnet_id = aws_subnet.cdp_public_subnets[0].id + + # Volume / root_block_device + root_block_device { + # Need to cast from string (yes) to bool + delete_on_termination = each.value.volume.delete_on_termination + volume_size = each.value.volume.volume_size + volume_type = each.value.volume.volume_type + } + + + associate_public_ip_address = true + + tags = merge(var.utility_vm_tags,{Name = each.value.name}) +} diff --git a/roles/infrastructure/files/aws/infra_aws_network.tf b/roles/infrastructure/files/aws/infra_aws_network.tf new file mode 100644 index 00000000..a0222309 --- /dev/null +++ b/roles/infrastructure/files/aws/infra_aws_network.tf @@ -0,0 +1,195 @@ +# ------- VPC ------- +# Create the VPC's +resource "aws_vpc" "cdp_vpc" { + cidr_block = var.vpc_cidr + tags = merge(var.env_tags,{Name = var.vpc_name}) + + instance_tenancy = "default" + enable_dns_support = true + enable_dns_hostnames = true +} + +# ------- AWS Public Network infrastructure ------- +# Internet Gateway +resource "aws_internet_gateway" "cdp_igw" { + vpc_id = aws_vpc.cdp_vpc.id + tags = merge(var.env_tags,{Name = var.igw_name}) +} + +# AWS VPC Public Subnets +resource "aws_subnet" "cdp_public_subnets" { + for_each = {for idx, subnet in var.public_subnets: idx => subnet} + + vpc_id = aws_vpc.cdp_vpc.id + cidr_block = each.value.cidr + map_public_ip_on_launch = true + availability_zone = each.value.az + tags = merge(var.env_tags,each.value.tags) +} + +# Public Route Table +resource "aws_default_route_table" "cdp_public_route_table" { + default_route_table_id = aws_vpc.cdp_vpc.default_route_table_id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.cdp_igw.id + } + + tags = merge(var.env_tags,{Name = var.public_route_table_name}) + +} + +# Associate the Public Route Table with the Public Subnets +resource "aws_route_table_association" "cdp_public_subnets" { + + for_each = aws_subnet.cdp_public_subnets + + subnet_id = each.value.id + route_table_id = aws_vpc.cdp_vpc.default_route_table_id +} + +# ------- AWS Private Networking infrastructure ------- + +# AWS VPC Private Subnets +resource "aws_subnet" "cdp_private_subnets" { + for_each = {for idx, subnet in var.private_subnets: idx => subnet} + + vpc_id = aws_vpc.cdp_vpc.id + cidr_block = each.value.cidr + map_public_ip_on_launch = false + availability_zone = each.value.az + tags = merge(var.env_tags,each.value.tags) +} + +# Private Route Table for the AWS VPC +# - Not implemeted in Terraform because of "when: no" in Ansible + +# Elastic IP for each NAT gateway +resource "aws_eip" "cdp_nat_gateway_eip" { + + for_each = {for idx, subnet in var.public_subnets: idx => subnet} + + vpc = true + tags = merge(var.env_tags,{Name = format("%s-%s-%02d", var.nat_gateway_name, "eip", index(var.public_subnets, each.value)+1)}) +} + +# Network Gateways (NAT) +resource "aws_nat_gateway" "cdp_nat_gateway" { + + # for_each = { for s in aws_subnet.cdp_public_subnets : s.id => s } + count = length(aws_subnet.cdp_public_subnets) + + subnet_id = aws_subnet.cdp_public_subnets[count.index].id + allocation_id = aws_eip.cdp_nat_gateway_eip[count.index].id + connectivity_type = "public" + + tags = merge(var.env_tags,{Name = format("%s-%02d", var.nat_gateway_name, count.index)}) + # tags = merge(var.env_tags,{Name = format("%s-%s-%02d", var.nat_gateway_name, "eip", index(aws_subnet.cdp_public_subnets, each.value)+1)}) +} + + +# Private Route Tables +resource "aws_route_table" "cdp_private_route_table" { + for_each = {for idx, subnet in var.private_subnets: idx => subnet} + + vpc_id = aws_vpc.cdp_vpc.id + + tags = merge(var.env_tags,{Name = format("%s-%02d", var.private_route_table_name, index(var.private_subnets, each.value))}) + + route { + cidr_block = "0.0.0.0/0" + #nat_gateway_id = aws_nat_gateway.cdp_nat_gateway[0].id + + nat_gateway_id = aws_nat_gateway.cdp_nat_gateway[(index(var.private_subnets, each.value) % length(aws_nat_gateway.cdp_nat_gateway))].id + + } +} + +# Associate the Private Route Tables with the Private Subnets +resource "aws_route_table_association" "cdp_private_subnets" { + + count = length(aws_subnet.cdp_private_subnets) + + subnet_id = aws_subnet.cdp_private_subnets[count.index].id + route_table_id = aws_route_table.cdp_private_route_table[count.index].id +} + +# ------- Security Groups ------- +# Default SG +resource "aws_security_group" "cdp_default_sg" { + vpc_id = aws_vpc.cdp_vpc.id + name = var.security_group_default_name + description = var.security_group_default_name + + tags = merge(var.env_tags,{Name = var.security_group_default_name}) + + # Create self reference ingress rule to allow + # communication among resources in the security group. + ingress { + from_port = 0 + to_port = 0 + protocol = "all" + self = true + } + + # Dynamic Block to create security group rule from var.sg_ingress + dynamic "ingress" { + for_each = var.security_group_rules_ingress + + content { + cidr_blocks = ingress.value.cidr + from_port = ingress.value.from_port + to_port = ingress.value.to_port + protocol = ingress.value.protocol + } + + } + + # Terraform removes the default ALLOW ALL egress. Let's recreate this + egress { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + protocol = "all" + } +} + +# Knox SG +resource "aws_security_group" "cdp_knox_sg" { + vpc_id = aws_vpc.cdp_vpc.id + name = var.security_group_knox_name + description = var.security_group_knox_name + + tags = merge(var.env_tags,{Name = var.security_group_knox_name}) + + # Create self reference ingress rule to allow + # communication among resources in the security group. + ingress { + from_port = 0 + to_port = 0 + protocol = "all" + self = true + } + + # Dynamic Block to create security group rule from var.sg_ingress + dynamic "ingress" { + for_each = var.security_group_rules_ingress + + content { + cidr_blocks = ingress.value.cidr + from_port = ingress.value.from_port + to_port = ingress.value.to_port + protocol = ingress.value.protocol + } + + } + + # Terraform removes the default ALLOW ALL egress. Let's recreate this + egress { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + protocol = "all" + } +} diff --git a/roles/infrastructure/files/aws/infra_aws_storage.tf b/roles/infrastructure/files/aws/infra_aws_storage.tf new file mode 100644 index 00000000..3a53443b --- /dev/null +++ b/roles/infrastructure/files/aws/infra_aws_storage.tf @@ -0,0 +1,66 @@ +# ------- S3 Buckets ------- +# NOTE: Variables cannot be used with the prevent_destroy lifecycle policy +# To overcome this limitation and still support infra__teardown_deletes_data = False +# we create two similar resource using a conditional on the for_each. +# Resource cdp_storage_delete_data is created when teardown_deletes_data = True +# Resource cdp_storage_retain_data is created when teardown_deletes_data = False +# References: +# * https://github.com/hashicorp/terraform/issues/22544#issuecomment-981575058 +# * https://discuss.hashicorp.com/t/conditionally-create-resources-when-a-for-each-loop-is-involved/20841/9 + +resource "aws_s3_bucket" "cdp_storage_delete_data" { + for_each = var.teardown_deletes_data ? toset(var.storage_locations[*].bucket) : [] + + bucket = each.value + tags = merge(var.env_tags,{Name = each.value}) + + # Purge storage locations during teardown? + force_destroy = true + lifecycle { + # A Terraform destroy of this resource will result in an error message. + prevent_destroy = false + } +} +resource "aws_s3_bucket" "cdp_storage_retain_data" { + for_each = var.teardown_deletes_data ? [] : toset(var.storage_locations[*].bucket) + + bucket = each.value + tags = merge(var.env_tags,{Name = each.value}) + + # Purge storage locations during teardown? + force_destroy = false + lifecycle { + # A Terraform destroy of this resource will result in an error message. + prevent_destroy = true + } +} + +# Separate bucket acl resource definition +resource "aws_s3_bucket_acl" "cdp_storage_acl" { + for_each = var.teardown_deletes_data ? aws_s3_bucket.cdp_storage_delete_data : aws_s3_bucket.cdp_storage_retain_data + + bucket = each.value.id + acl = "private" +} + +# ------- AWS Buckets directory structures ------- +resource "aws_s3_object" "cdp_storage_object" { + + for_each = {for idx, object in var.storage_locations: idx => object} + + # Bucket is either from 'cdp_storage_delete_data' or 'cdp_storage_retain_data' resource depending on teardown_deletes_data' + bucket = var.teardown_deletes_data ? aws_s3_bucket.cdp_storage_delete_data[each.value.bucket].id : aws_s3_bucket.cdp_storage_retain_data[each.value.bucket].id + + key = each.value.object + content_type = "application/x-directory" +} + +# ------- Download Mirror Bucket ------- +# TODO: Don't fail if Mirror Bucket already exists from non-Terraform code. +# resource "aws_s3_bucket" "cdp_utility_bucket" { +# bucket = var.utility_bucket +# acl = "private" +# force_destroy = true + +# tags = merge(var.env_tags,{Name = var.utility_bucket}) +# } diff --git a/roles/infrastructure/template/aws/provider.tf.j2 b/roles/infrastructure/files/aws/provider.tf similarity index 100% rename from roles/infrastructure/template/aws/provider.tf.j2 rename to roles/infrastructure/files/aws/provider.tf diff --git a/roles/infrastructure/files/aws/terraform_variables.tf b/roles/infrastructure/files/aws/terraform_variables.tf new file mode 100644 index 00000000..d72345c9 --- /dev/null +++ b/roles/infrastructure/files/aws/terraform_variables.tf @@ -0,0 +1,195 @@ +# ------- Global settings ------- +variable "aws_profile" { + type = string + description = "Profile for AWS cloud credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +variable "region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_tags" { + type = map + description = "Tags applied to provised resources" + + default = { + comment = "Created with Terraform by cloudera-deploy" + } +} + +# ------- Network Resources ------- +variable "vpc_name" { + type = string + description = "VPC name" +} + +variable "vpc_cidr" { + type = string + description = "VPC CIDR Block" +} + +variable "igw_name" { + type = string + description = "Internet Gateway" +} + +# Public Network infrastructure +variable "public_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Public Subnets" + default = [] +} + +variable "public_route_table_name" { + type = string + description = "Public Route Table Name" +} + +# Private Network infrastructure +variable "private_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Private Subnets" + default = [] +} + +variable "nat_gateway_name" { + type = string + + description = "Nat Gateway" + default = "CDP_NAT_Gateway" +} + +variable "private_route_table_name" { + type = string + + description = "Private Route Table" + default = "CDP_Private_RT" +} + +# Security Groups +variable "security_group_default_name" { + type = string + + description = "Default Security Group for CDP environment" +} + +variable "security_group_knox_name" { + type = string + + description = "Knox Security Group for CDP environment" +} + +variable "security_group_rules_ingress" { + type = list(object({ + cidr = list(string) + from_port = string + to_port = string + protocol = string + })) + + description = "Ingress rules for Security Group" + default = [] +} + +# ------- Storage Resources ------- +variable "storage_locations" { + type = list(object({ + bucket = string + object = string + })) + + description = "Storage locations for CDP environment" +} + +variable "teardown_deletes_data" { + type = bool + + description = "Purge storage locations during teardown" +} + +variable "utility_bucket" { + type = string + + description = "Utility bucket used as a mirror for downloaded PvC parcels" +} + +# ------- Compute Resources ------- +# Dynamic Inventory VMs +variable "dynamic_inventory_vms" { + type = list(object({ + name = string + instance_type = string + ami = string + volume = object({ + delete_on_termination = bool + volume_size = string + volume_type = string + }) + })) + + description = "List of VMs to be created as part of dynamic inventory" + + default = [] +} + +variable "dynamic_inventory_tags" { + type = map + description = "Tags applied to provisioned dynamic inventory resources" + + default = {} +} + +variable "dynamic_inventory_public_key_id" { + type = string + + description = "Name of the Public SSH key for the dynamic inventory VMs" +} + +# Localised Utility VM +variable "utility_vms" { + type = list(object({ + name = string + instance_type = string + ami = string + volume = object({ + delete_on_termination = bool + volume_size = string + volume_type = string + }) + })) + + description = "Utility VMs used for hosting parcel mirror" + + default = [] +} + +variable "utility_vm_tags" { + type = map + description = "Tags applied to Utility VM" + + default = {} +} + +variable "utility_vm_public_key_id" { + type = string + + description = "Name of the Public SSH key for the Utility VM" + + default = "" +} diff --git a/roles/infrastructure/tasks/initialize_aws_terraform.yml b/roles/infrastructure/tasks/initialize_aws_terraform.yml index 6c02183a..2007318f 100644 --- a/roles/infrastructure/tasks/initialize_aws_terraform.yml +++ b/roles/infrastructure/tasks/initialize_aws_terraform.yml @@ -22,43 +22,50 @@ path: "{{ infra__terraform_artefact_dir }}/infra" state: directory -# Apply template for Terraform provider -- name: Generate Terraform Provider - ansible.builtin.template: - src: 'template/{{ infra__type }}/provider.tf.j2' +# Copy Terraform provider file +- name: Copy Terraform Provider file + ansible.builtin.copy: + src: 'files/{{ infra__type }}/provider.tf' dest: "{{ infra__terraform_template_dir }}/infra/provider.tf" -# Apply template for Terraform backend state -- name: Generate Terraform Backend State - ansible.builtin.template: - src: 'template/{{ infra__type }}/backend_state.tf.j2' - dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" - -# Apply template for Terraform variables -- name: Generate Terraform Variables - ansible.builtin.template: - src: 'template/{{ infra__type }}/terraform_variables.tf.j2' +# Copy Terraform variables file +- name: Copy Terraform Variables declaration file + ansible.builtin.copy: + src: 'files/{{ infra__type }}/terraform_variables.tf' dest: "{{ infra__terraform_template_dir }}/infra/variables.tf" no_log: false -# Apply template for Terraform infra.... +# Copy the Terraform resource files.... # ...network resources -- name: Generate Terraform infra file for network resources - ansible.builtin.template: - src: 'template/{{ infra__type }}/infra_{{ infra__type }}_network.tf.j2' +- name: Copy Terraform resource file for network resources + ansible.builtin.copy: + src: 'files/{{ infra__type }}/infra_{{ infra__type }}_network.tf' dest: "{{ infra__terraform_template_dir }}/infra/infra_network.tf" no_log: false # ...storage resources -- name: Generating Terraform infra file for storage resources - ansible.builtin.template: - src: 'template/{{ infra__type }}/infra_{{ infra__type }}_storage.tf.j2' +- name: Copy Terraform resource file for storage resources + ansible.builtin.copy: + src: 'files/{{ infra__type }}/infra_{{ infra__type }}_storage.tf' dest: "{{ infra__terraform_template_dir }}/infra/infra_storage.tf" no_log: false # ...compute resources - name: Generating Terraform infra file for compute resources - ansible.builtin.template: - src: 'template/{{ infra__type }}/infra_{{ infra__type }}_compute.tf.j2' + ansible.builtin.copy: + src: 'files/{{ infra__type }}/infra_{{ infra__type }}_compute.tf' dest: "{{ infra__terraform_template_dir }}/infra/infra_compute.tf" no_log: false + +# Apply template for Terraform backend state +- name: Generate Terraform Backend State + ansible.builtin.template: + src: 'templates/{{ infra__type }}/backend_state.tf.j2' + dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" + +# Create Terraform variable definitions from template +- name: Generate Terraform Variables definition + ansible.builtin.template: + src: 'templates/{{ infra__type }}/terraform.tfvars.j2' + dest: "{{ infra__terraform_template_dir }}/infra/terraform.tfvars" + no_log: false diff --git a/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml b/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml index 5a0545a3..73b244f3 100644 --- a/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml +++ b/roles/infrastructure/tasks/initialize_teardown_aws_terraform.yml @@ -17,21 +17,21 @@ path: "{{ infra__terraform_template_dir }}/infra" state: directory -# Apply template for Terraform provider -- name: Generate Terraform Provider - ansible.builtin.template: - src: 'template/{{ infra__type }}/provider.tf.j2' +# Copy Terraform provider file +- name: Copy Terraform Provider file + ansible.builtin.copy: + src: 'files/{{ infra__type }}/provider.tf' dest: "{{ infra__terraform_template_dir }}/infra/provider.tf" +# Copy Terraform variables file +- name: Copy Terraform Variables declaration file + ansible.builtin.copy: + src: 'files/{{ infra__type }}/terraform_variables.tf' + dest: "{{ infra__terraform_template_dir }}/infra/variables.tf" + no_log: false + # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'template/{{ infra__type }}/backend_state.tf.j2' - dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" - -# Apply template for Terraform variables -- name: Generate Terraform Variables - ansible.builtin.template: - src: 'template/{{ infra__type }}/terraform_variables.tf.j2' - dest: "{{ infra__terraform_template_dir }}/infra/variables.tf" - no_log: false + src: 'templates/{{ infra__type }}/backend_state.tf.j2' + dest: "{{ infra__terraform_template_dir }}/infra/backend_state.tf" \ No newline at end of file diff --git a/roles/infrastructure/template/aws/infra_aws_compute.tf.j2 b/roles/infrastructure/template/aws/infra_aws_compute.tf.j2 deleted file mode 100644 index f1eff746..00000000 --- a/roles/infrastructure/template/aws/infra_aws_compute.tf.j2 +++ /dev/null @@ -1,62 +0,0 @@ -{# *** Set the __aws_subnets to use for the aws_instance subnet_id parameter ****#} -{% if not infra__tunnel %} -{% set __aws_subnets = (__aws_subnets | default([])) | union(infra__vpc_public_subnets_info) %} -{% endif %} - -{% if ( infra__tunnel ) and ( infra__aws_subnet_ids is not defined ) %} -{% set __aws_subnets = (__aws_subnets | default([])) | union(infra__vpc_public_subnets_info) | union(infra__vpc_private_subnets_info)%} -{% endif %} - -# ------- Dynamic Inventory VMs ------- -{% for __infra_compute_instance_item in range(0, infra__dynamic_inventory_count | int ) | list %} -resource "aws_instance" "{{ '-'.join([infra__namespace, infra__dynamic_inventory_vm_suffix, '%02d' | format(__infra_compute_instance_item)]) }}" { - vpc_security_group_ids = [aws_security_group.{{ infra__security_group_default_name }}.id] - key_name = "{{ infra__public_key_id }}" - instance_type = "{{ infra__dynamic_inventory_vm_type_default[infra__type][infra__dynamic_inventory_vm_type] }}" - ami = "{{ __infra_aws_ami_info.image_id }}" - ebs_optimized = true - - # Volume / root_block_device - root_block_device { - # Need to cast from string (yes) to bool - delete_on_termination = {{ infra__dynamic_inventory_delete_storage | bool | lower }} - volume_size = "{{ infra__dynamic_inventory_storage_size }}" - volume_type = "{{ infra__dynamic_inventory_storage_type_default[infra__type][infra__dynamic_inventory_storage_type] }}" - } - - subnet_id = aws_subnet.{{ __aws_subnets | map(attribute='name') | first }}.id - - associate_public_ip_address = true - - {# to_json is used to convert single quotes around IP to double #} - tags = merge({{ infra__dynamic_inventory_tags | to_json }},{Name = "{{ '-'.join([infra__namespace, infra__dynamic_inventory_vm_suffix, infra__dynamic_inventory_os[::2], '%02d' | format(__infra_compute_instance_item)]) }}"}) - -} -{% endfor %} - -{% if infra__create_utility_service %} -# ------- Localised Utility VM Instance ------- -resource "aws_instance" "{{ '-'.join([infra__namespace, infra__region, 'utility_vm' ]) }}" { - vpc_security_group_ids = [aws_security_group.{{ infra__security_group_default_name }}.id] - key_name = "{{ infra__public_key_id }}" - instance_type = "{{ infra__dynamic_inventory_vm_type_default[infra__type]['sml'] }}" - ami = "{{ __infra_aws_ami_info.image_id }}" - ebs_optimized = true - - # Volume / root_block_device - root_block_device { - # Need to cast from string (yes) to bool - delete_on_termination = {{ infra__dynamic_inventory_delete_storage | bool | lower }} - volume_size = 100 - volume_type = "{{ infra__dynamic_inventory_storage_type_default[infra__type]['std'] }}" - } - - subnet_id = aws_subnet.{{ __aws_subnets | map(attribute='name') | first }}.id - - associate_public_ip_address = true - - {# to_json is used to convert single quotes around IP to double #} - tags = merge({{ infra__dynamic_inventory_tags | to_json }},{Name = "{{ '-'.join([infra__namespace, infra_region, 'utility_vm' ]) }}"}) -} - -{% endif %} \ No newline at end of file diff --git a/roles/infrastructure/template/aws/infra_aws_network.tf.j2 b/roles/infrastructure/template/aws/infra_aws_network.tf.j2 deleted file mode 100644 index 97d9bdc8..00000000 --- a/roles/infrastructure/template/aws/infra_aws_network.tf.j2 +++ /dev/null @@ -1,236 +0,0 @@ -# ------- VPC ------- -# Create the VPC's -resource "aws_vpc" "{{ infra__vpc_name }}" { - cidr_block = "{{ infra__vpc_cidr }}" - tags = merge(var.env_tags,{Name = "{{ infra__vpc_name }}"}) - - instance_tenancy = "default" - enable_dns_support = true - enable_dns_hostnames = true -} - -{# *** START Create AWS Public Network infrastructure ****#} -{% if infra__aws_subnet_ids is not defined %} -# ------- AWS Public Network infrastructure ------- -# Internet Gateway -resource "aws_internet_gateway" "{{ infra__aws_igw_name }}" { - vpc_id = aws_vpc.{{ infra__vpc_name }}.id - tags = merge(var.env_tags,{Name = "{{ infra__aws_igw_name }}"}) -} - -# AWS VPC Public Subnets -{% for __aws_public_subnet_item in infra__vpc_public_subnets_info %} -resource "aws_subnet" "{{ __aws_public_subnet_item.name }}" { - vpc_id = aws_vpc.{{ infra__vpc_name }}.id - cidr_block = "{{ __aws_public_subnet_item.cidr }}" - map_public_ip_on_launch = true - availability_zone = "{{ __aws_az_info.availability_zones[loop.index0 % infra__aws_vpc_az_count | int].zone_name }}" - tags = merge(var.env_tags,{% for key, value in __aws_public_subnet_item.tags.items() %}{ "{{ key }}" = "{{ value }}" },{% endfor %}) -} -{% endfor %} - -# Public Route Table -resource "aws_default_route_table" "{{ infra__aws_public_route_table_name }}" { - default_route_table_id = aws_vpc.{{ infra__vpc_name }}.default_route_table_id - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.{{ infra__aws_igw_name }}.id - } - - tags = merge(var.env_tags,{Name = "{{ infra__aws_public_route_table_name }}"}) - -} - -# Associate the Public Route Table with the Public Subnets -{% for __aws_public_subnet_item in infra__vpc_public_subnets_info %} -resource "aws_route_table_association" "{{ __aws_public_subnet_item.name }}-association" { - subnet_id = aws_subnet.{{ __aws_public_subnet_item.name }}.id - route_table_id = aws_vpc.{{ infra__vpc_name }}.default_route_table_id -} -{% endfor %} - -{% endif %} -{# *** END Create AWS Public Network infrastructure ****#} - - -{# *** START Create AWS Private Network infrastructure ****#} -{% if ( infra__tunnel ) and ( infra__aws_subnet_ids is not defined ) %} - -# ------- AWS Private Networking infrastructure ------- - -# AWS VPC Private Subnets -{% for __aws_private_subnet_item in infra__vpc_private_subnets_info %} -resource "aws_subnet" "{{ __aws_private_subnet_item.name }}" { - vpc_id = aws_vpc.{{ infra__vpc_name }}.id - cidr_block = "{{ __aws_private_subnet_item.cidr }}" - map_public_ip_on_launch = false - availability_zone = "{{ __aws_az_info.availability_zones[loop.index0 % infra__aws_vpc_az_count | int].zone_name }}" - tags = merge(var.env_tags,{% for key, value in __aws_private_subnet_item.tags.items() %}{ "{{ key }}" = "{{ value }}" },{% endfor %}) -} -{% endfor %} - -# Private Route Table for the AWS VPC -# - Not implemeted in Terraform because of "when: no" in Ansible - -# Elastic IP for each NAT gateway -{% for __aws_public_subnet_item in infra__vpc_public_subnets_info %} -resource "aws_eip" "{{ infra__aws_nat_gateway_name }}-eip-{{ loop.index0 }}" { - vpc = true - - tags = var.env_tags -} -{% endfor %} - -# Network Gateways (NAT) -{% for __aws_public_subnet_item in infra__vpc_public_subnets_info %} -resource "aws_nat_gateway" "{{ infra__aws_nat_gateway_name }}-{{ loop.index0 }}" { - - subnet_id = aws_subnet.{{ __aws_public_subnet_item.name }}.id - allocation_id = aws_eip.{{ infra__aws_nat_gateway_name }}-eip-{{ loop.index0 }}.id - connectivity_type = "public" - - tags = merge(var.env_tags,{Name = "{{ '-'.join([infra__aws_nat_gateway_name, loop.index0 | string ]) }}"}) -} -{% endfor %} - -# Private Route Tables -{% for __aws_private_subnet_item in infra__vpc_private_subnets_info %} -resource "aws_route_table" "{{ infra__aws_private_route_table_name }}-{{ loop.index0 }}" { - vpc_id = aws_vpc.{{ infra__vpc_name }}.id - - tags = merge(var.env_tags,{Name = "{{ '-'.join([infra__aws_private_route_table_name, loop.index0 | string ]) }}"}) - - route { - cidr_block = "0.0.0.0/0" - nat_gateway_id = aws_nat_gateway.{{ infra__aws_nat_gateway_name }}-{{ loop.index0 % infra__vpc_public_subnets_info | length }}.id - } - -} -{% endfor %} - -# Associate the Private Route Tables with the Private Subnets -{% for __aws_private_subnet_item in infra__vpc_private_subnets_info %} -resource "aws_route_table_association" "{{ infra__aws_private_route_table_name }}-{{ loop.index0 }}-association" { - subnet_id = aws_subnet.{{ __aws_private_subnet_item.name }}.id - route_table_id = aws_route_table.{{ infra__aws_private_route_table_name }}-{{ loop.index0 }}.id -} -{% endfor %} - -{% endif %} -{# *** END Create AWS PRIVATE Network infrastructure ****#} - -# ------- Security Groups ------- -{% set __security_group_names = [infra__security_group_knox_name, infra__security_group_default_name] %} - -{% for __security_group_name_item in __security_group_names %} -resource "aws_security_group" "{{ __security_group_name_item }}" { - vpc_id = aws_vpc.{{ infra__vpc_name }}.id - name = "{{ __security_group_name_item }}" - description = "{{ __security_group_name_item }}" - - tags = merge(var.env_tags,{Name = "{{ __security_group_name_item }}"}) - - # Create self reference ingress rule to allow - # communication among resources in the security group. - ingress { - from_port = 0 - to_port = 0 - protocol = "all" - self = true - } - -{# ******* NOTE: HANDLING OF SECURITY GROUP RULES *******#} -{# Need to loop over each security group rule:#} -{# Need to Process & Translate *CIDR Block* according to following rules:#} -{# * Can be empty in which case remove entry - DONE#} -{# * Can be a string. Check if this works; if not cast to list element#} -{# * Can be a list. Use directly - DONE #} - -{# Need to Process & Translate *Ports* according to following rules: #} -{# * If proto is `all` (or -1) then ports are 0 - DONE#} -{# * If toPort and fromPort specified use those#} -{# * ports: can be a range. Split and select - DONE#} -{# * ports: could be a list. Loop over each - DONE #} -{# * If no ports, toPort and fromPort then use default of full range (0)#} - -{# **Loop over security group rule**#} -{% for ingress in infra__aws_security_group_rules %} -# ----- Raw Inputs ----- -# ports = {{ ingress.ports|pprint }} -# cidr_blocks = {{ ingress.cidr_ip }} -# protocol = {{ ingress.proto }} -# ---------------------- -{# **Process CIDR Block**#} -{# 1. CIDR block can be empty, in which case remove entry#} -{% if ingress.cidr_ip | length > 0 %} -{# 1a. If CIDR is a single string value then we need to convert single item list #} -{# to_json is used to convert single quotes around IP to double #} -{% if ingress.cidr_ip is string %} -{% set cidr_ingress = 'cidr_blocks = ' + [ingress.cidr_ip]| to_json | string %} -{% else %} -{# 1b. If CIDR is a list to convert single quotes to double (to_json) and stringify #} -{% set cidr_ingress = 'cidr_blocks = ' + ingress.cidr_ip| to_json | string %} -{% endif %} -{% else %} -{# 1c. If CIDR is empty we remove the entry #} -{% set cidr_ingress = '' %} -{% endif %} -{# **Process Ports**#} -{# 1. If proto is 'all' or -1 #} -{% if ingress.proto in ['all', -1] or ingress.ports is not defined %} -{# 1a. to and from ports are 0 #} -{% set toPort = 0 %} -{% set fromPort = 0 %} -{# 1b. Print out ingress #} - ingress { - {{ cidr_ingress }} - from_port = "{{ toPort }}" - to_port = "{{ fromPort }}" - protocol = "{{ ingress.proto }}" - } -{# 2. Ports can be a String (either a single value or a range)#} -{% elif ingress.ports is string %} -{# 2a. Attempt to split the string at '-' to create a list#} -{% set portList = ingress.ports.split('-') %} -{# 2b. In the list toPort is the first element#} -{% set toPort = portList[0] %} -{# 2c. Value of fromPort depends on length of the split string.#} -{% if portList|length > 1 %} -{# ...it's the second element if a range is given (i.e. length > 1)#} -{% set fromPort = portList[1] %} -{% else %} -{# ...it's also the first element if a single value is given#} -{% set fromPort = portList[0] %} -{% endif %} -{# 2d. Print the ingress block#} - ingress { - {{ cidr_ingress }} - from_port = "{{ toPort }}" - to_port = "{{ fromPort }}" - protocol = "{{ ingress.proto }}" - } -{# 3. Ports can be a List of individual ports which we need to loop over#} -{% elif ingress.ports is iterable and (ingress.ports is not string and ingress.ports is not mapping) %} -{# 3a. Loop over each port separately and print ingress #} -{% for port in ingress.ports %} - ingress { - {{ cidr_ingress }} - from_port = "{{ port }}" - to_port = "{{ port }}" - protocol = "{{ ingress.proto }}" - } -{% endfor %} -{% endif %} -{% endfor %} - - # Terraform removes the default ALLOW ALL egress. Let's recreate this - egress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - to_port = 0 - protocol = "all" - } - -} -{% endfor %} diff --git a/roles/infrastructure/template/aws/infra_aws_storage.tf.j2 b/roles/infrastructure/template/aws/infra_aws_storage.tf.j2 deleted file mode 100644 index 1f0188a5..00000000 --- a/roles/infrastructure/template/aws/infra_aws_storage.tf.j2 +++ /dev/null @@ -1,49 +0,0 @@ -# ------- S3 Buckets ------- -# Get unique list of buckets from infra__aws_storage_locations -{% for __aws_storage_location_item in ( infra__aws_storage_locations | map(attribute='bucket') | list | unique ) %} -resource "aws_s3_bucket" "{{ __aws_storage_location_item }}" { - bucket = "{{ __aws_storage_location_item }}" - -{% if infra__teardown_deletes_data %} - force_destroy = true -{% else %} - {# TODO: How to skip teardown of this resource if infra__teardown_deletes_data is False #} - lifecycle { - # A Terraform destroy of this resource will result in an error message. - prevent_destroy = true - } -{% endif %} - - tags = merge(var.env_tags,{Name = "{{ __aws_storage_location_item }}"}) -} - -# Separate bucket acl resource definition -resource "aws_s3_bucket_acl" "{{ __aws_storage_location_item }}" { - bucket = aws_s3_bucket.{{ __aws_storage_location_item }}.id - acl = "private" -} -{% endfor %} - -# ------- AWS Buckets directory structures ------- -{% for __aws_storage_object_item in infra__aws_storage_locations %} -{# Terraform resources cannot have '/' so replace with '_' #} -{% set __aws_storage_object_resource = __aws_storage_object_item.path |replace("/", "_") %} - -resource "aws_s3_object" "{{ __aws_storage_object_resource }}" { - bucket = aws_s3_bucket.{{ __aws_storage_object_item.bucket }}.id - key = "{{ __aws_storage_object_item.path }}/" - # Below may not be required once we have the '/' - content_type = "application/x-directory" -} -{% endfor %} - -# ------- Download Mirror Bucket ------- -# TODO: Don't fail if Mirror Bucket already exists from non-Terraform code. -# resource "aws_s3_bucket" "{{ infra__utlity_bucket_name }}" { -# bucket = "{{ infra__utlity_bucket_name }}" -# acl = "private" -# force_destroy = true - -# tags = merge(var.env_tags,{Name = "{{ infra__utlity_bucket_name }}"}) -# } - diff --git a/roles/infrastructure/template/aws/terraform_variables.tf.j2 b/roles/infrastructure/template/aws/terraform_variables.tf.j2 deleted file mode 100644 index 8c5cc41e..00000000 --- a/roles/infrastructure/template/aws/terraform_variables.tf.j2 +++ /dev/null @@ -1,19 +0,0 @@ -variable "aws_profile" { - # Profile is default is plat__aws_profile is blank - default = "{{ (infra__aws_profile | length > 0) | ternary(infra__aws_profile, 'default') }}" -} - -# Region -variable "region" { - default = "{{ infra__region }}" -} - - -variable "env_tags" { - default = { - {% for key, value in infra__tags.items() %} - {{ key }} = "{{ value }}" - {% endfor %} - comment = "Created with Terraform by cloudera-deploy" - } -} diff --git a/roles/infrastructure/template/aws/backend_state.tf.j2 b/roles/infrastructure/templates/aws/backend_state.tf.j2 similarity index 85% rename from roles/infrastructure/template/aws/backend_state.tf.j2 rename to roles/infrastructure/templates/aws/backend_state.tf.j2 index 0b7f5d16..cba5cf0a 100644 --- a/roles/infrastructure/template/aws/backend_state.tf.j2 +++ b/roles/infrastructure/templates/aws/backend_state.tf.j2 @@ -7,7 +7,7 @@ terraform { backend "s3" { region = "{{ infra__region }}" bucket = "{{ infra__terraform_remote_state_bucket }}" - key = "{{infra__namespace}}/infra/terraform.tfstate" + key = "{{ infra__namespace }}/infra/terraform.tfstate" dynamodb_table = "{{ infra__terraform_remote_state_lock_table }}" } } diff --git a/roles/infrastructure/templates/aws/terraform.tfvars.j2 b/roles/infrastructure/templates/aws/terraform.tfvars.j2 new file mode 100644 index 00000000..a9836da1 --- /dev/null +++ b/roles/infrastructure/templates/aws/terraform.tfvars.j2 @@ -0,0 +1,199 @@ +# ------- Global settings ------- +region = "{{ infra__region }}" + +aws_profile = "{{ (infra__aws_profile | length > 0) | ternary(infra__aws_profile, 'default') }}" + +env_tags = { +{% for key, value in infra__tags.items() %} + {{ key }} = "{{ value }}" +{% endfor %} + comment = "Created with Terraform by cloudera-deploy" +} + +# ------- Network Resources ------- +vpc_name = "{{ infra__vpc_name }}" +vpc_cidr = "{{ infra__vpc_cidr }}" +igw_name = "{{ infra__aws_igw_name }}" + +# Public Network infrastructure +{% if infra__aws_subnet_ids is not defined %} +public_subnets = [ +{% for __aws_public_subnet_item in infra__vpc_public_subnets_info %} +{ + name = "{{ __aws_public_subnet_item.name }}", + cidr = "{{ __aws_public_subnet_item.cidr }}", + az = "{{ __aws_az_info.availability_zones[loop.index0 % infra__aws_vpc_az_count | int].zone_name }}", + tags = { + {% for key, value in __aws_public_subnet_item.tags.items() %} + "{{ key }}" = "{{ value }}", + {% endfor %} + }, +}, +{% endfor %} +] + +public_route_table_name = "{{ infra__aws_public_route_table_name }}" +{% endif %} + + +# Private Network infrastructure +{% if ( infra__tunnel ) and ( infra__aws_subnet_ids is not defined ) %} +private_subnets = [ +{% for __aws_private_subnet_item in infra__vpc_private_subnets_info %} +{ + name = "{{ __aws_private_subnet_item.name }}", + cidr = "{{ __aws_private_subnet_item.cidr }}", + az = "{{ __aws_az_info.availability_zones[loop.index0 % infra__aws_vpc_az_count | int].zone_name }}", + tags = { + {% for key, value in __aws_private_subnet_item.tags.items() %} + "{{ key }}" = "{{ value }}", + {% endfor %} + }, +}, +{% endfor %} +] + +nat_gateway_name = "{{ infra__aws_nat_gateway_name }}" + +private_route_table_name = "{{ infra__aws_private_route_table_name }}" + +{% endif %} + +# Security Groups +security_group_default_name = "{{ infra__security_group_default_name }}" +security_group_knox_name = "{{ infra__security_group_knox_name }}" + +security_group_rules_ingress = [ +{# ******* NOTE: HANDLING OF SECURITY GROUP RULES *******#} +{# Need to loop over each security group rule and...#} +{# 1/ Process & Translate *CIDR Block* according to following rules:#} +{# * Can be empty in which case remove entry#} +{# * Can be a string. Check if this works; if not cast to list element#} +{# * Can be a list. Use directly #} + +{# 2/ Process & Translate *Ports* according to following rules: #} +{# * If proto is `all` (or -1) then ports are 0#} +{# * If toPort and fromPort specified use those#} +{# * ports: can be a range. Split and select#} +{# * ports: could be a list. Loop over each #} +{# * If no ports, toPort and fromPort then use default of full range (0)#} +{# ******* END NOTE *******#} +{% for ingress in infra__aws_security_group_rules %} +{# ** 1/ Process CIDR Block**#} +{# 1. CIDR block can be empty, in which case remove entry#} +{% if ingress.cidr_ip | length > 0 %} +{# 1a. If CIDR is a single string value then we need to convert single item list #} +{# to_json is used to convert single quotes around IP to double #} +{% if ingress.cidr_ip is string %} +{% set cidr_ingress = [ingress.cidr_ip]| to_json | string %} +{% else %} +{# 1b. If CIDR is a list to convert single quotes to double quotes (to_json) and stringify #} +{% set cidr_ingress = ingress.cidr_ip| to_json | string %} +{% endif %} +{% else %} +{# 1c. If CIDR is empty we remove the entry #} +{% set cidr_ingress = null %} +{% endif %} + +{# **Process Ports**#} +{# 1. If proto is 'all' or -1 #} +{% if ingress.proto in ['all', -1] or ingress.ports is not defined %} +{# 1a. to and from ports are 0 #} +{% set toPort = 0 %} +{% set fromPort = 0 %} +{# 1b. Print out ingress #} +{ + cidr = {{ cidr_ingress }} + from_port = "{{ toPort }}" + to_port = "{{ fromPort }}" + protocol = "{{ ingress.proto }}" +}, +{# 2. Ports can be a String (either a single value or a range)#} +{% elif ingress.ports is string %} +{# 2a. Attempt to split the string at '-' to create a list#} +{% set portList = ingress.ports.split('-') %} +{# 2b. In the list toPort is the first element#} +{% set toPort = portList[0] %} +{# 2c. Value of fromPort depends on length of the split string.#} +{% if portList|length > 1 %} +{# ...it's the second element if a range is given (i.e. length > 1)#} +{% set fromPort = portList[1] %} +{% else %} +{# ...it's also the first element if a single value is given#} +{% set fromPort = portList[0] %} +{% endif %} +{# 2d. Print the ingress block#} +{ + cidr = {{ cidr_ingress }} + from_port = "{{ toPort }}" + to_port = "{{ fromPort }}" + protocol = "{{ ingress.proto }}" +}, +{# 3. Ports can be a List of individual ports which we need to loop over#} +{% elif ingress.ports is iterable and (ingress.ports is not string and ingress.ports is not mapping) %} +{# 3a. Loop over each port separately and print ingress #} +{% for port in ingress.ports %} +{ + cidr = {{ cidr_ingress }} + from_port = "{{ port }}" + to_port = "{{ port }}" + protocol = "{{ ingress.proto }}" +}, +{% endfor %} +{% endif %} +{% endfor %} +] + +# ------- Storage Resources ------- +storage_locations = [ +{% for __aws_storage_object_item in infra__aws_storage_locations %} +{ + bucket = "{{ __aws_storage_object_item.bucket }}", + object = "{{ __aws_storage_object_item.path }}/", {# Append a / to ensure a directory is created #} +}, +{% endfor %} +] +teardown_deletes_data = "{{ infra__teardown_deletes_data | lower }}" + +utility_bucket = "{{ infra__utlity_bucket_name }}" + +# ------- Compute Resources ------- +# Dynamic Inventory VMs +dynamic_inventory_vms = [ +{% for __infra_compute_instance_item in range(0, infra__dynamic_inventory_count | int ) | list %} +{ + name = "{{ '-'.join([infra__namespace, infra__dynamic_inventory_vm_suffix, infra__dynamic_inventory_os[::2], '%02d' | format(__infra_compute_instance_item)]) }}", + instance_type = "{{ infra__dynamic_inventory_vm_type_default[infra__type][infra__dynamic_inventory_vm_type] }}", + ami = "{{ __infra_aws_ami_info.image_id }}", + volume = { + delete_on_termination = {{ infra__dynamic_inventory_delete_storage | bool | lower }}, + volume_size = {{ infra__dynamic_inventory_storage_size }}, + volume_type = "{{ infra__dynamic_inventory_storage_type_default[infra__type][infra__dynamic_inventory_storage_type] }}", + } +}, +{% endfor %} +] + +{# to_json is used to convert single quotes around tags to double #} +dynamic_inventory_tags = {{ infra__dynamic_inventory_tags | to_json }} + +dynamic_inventory_public_key_id = "{{ infra__public_key_id }}" + +# Localised Utility VM - if requested +{% if infra__create_utility_service %} +utility_vms = [ +{ + name = "{{ '-'.join([infra__namespace, infra__region, 'utility_vm' ]) }}", + instance_type = "{{ infra__dynamic_inventory_vm_type_default[infra__type]['sml'] }}", + ami = "{{ __infra_aws_ami_info.image_id }}", + volume = + delete_on_termination = {{ infra__dynamic_inventory_delete_storage | bool | lower }}, + volume_size = 100, + volume_type = "{{ infra__dynamic_inventory_storage_type_default[infra__type][infra__dynamic_inventory_storage_type] }}", + } +} +] +{% endif %} + +utility_vm_tags = {{ infra__dynamic_inventory_tags | to_json }} +utility_vm_public_key_id = "{{ infra__public_key_id }}" \ No newline at end of file diff --git a/roles/platform/files/aws/plat_aws_authz_policies.tf b/roles/platform/files/aws/plat_aws_authz_policies.tf new file mode 100644 index 00000000..9ae40248 --- /dev/null +++ b/roles/platform/files/aws/plat_aws_authz_policies.tf @@ -0,0 +1,74 @@ +# ------- AWS Cross Account Policy ------- +# The policy here is a dict variable so we'll use the variable +# directly in the aws_iam_policy resource. +resource "aws_iam_policy" "cdp_xaccount_policy" { + name = var.xaccount_policy_name + description = "CDP Cross Account policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.xaccount_policy_name}) + + policy = var.xaccount_account_policy_doc +} + +# ------- CDP IDBroker Assume Role policy ------- +# First create the assume role policy document +data "aws_iam_policy_document" "cdp_idbroker_policy_doc" { + version = "2012-10-17" + + statement { + sid = "VisualEditor0" + actions = ["sts:AssumeRole"] + effect = "Allow" + resources = ["*"] + } +} + +# Then create the policy using the document +resource "aws_iam_policy" "cdp_idbroker_policy" { + name = var.idbroker_policy_name + description = "CDP IDBroker Assume Role policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.idbroker_policy_name}) + + policy = data.aws_iam_policy_document.cdp_idbroker_policy_doc.json +} + +# ------- CDP Data Access Policies - Log ------- +resource "aws_iam_policy" "cdp_log_data_access_policy" { + name = var.log_data_access_policy_name + description = "CDP Log Location Access policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.log_data_access_policy_name}) + + policy = file(var.log_data_access_policy_doc) +} + +# ------- CDP Data Access Policies - ranger_audit_s3 ------- +resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { + name = var.ranger_audit_s3_policy_name + description = "CDP Ranger Audit S3 Access policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.ranger_audit_s3_policy_name}) + + policy = file(var.ranger_audit_s3_policy_doc) +} + +# ------- CDP Data Access Policies - datalake_admin_s3 ------- +resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { + name = var.datalake_admin_s3_policy_name + description = "CDP Datalake Admin S3 Access policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.datalake_admin_s3_policy_name}) + + policy = file(var.datalake_admin_s3_policy_doc) +} + +# ------- CDP Data Access Policies - bucket_access ------- +resource "aws_iam_policy" "cdp_bucket_data_access_policy" { + name = var.bucket_access_policy_name + description = "CDP Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(var.env_tags,{Name = var.bucket_access_policy_name}) + + policy = file(var.bucket_access_policy_doc) +} diff --git a/roles/platform/files/aws/plat_aws_authz_roles.tf b/roles/platform/files/aws/plat_aws_authz_roles.tf new file mode 100644 index 00000000..bdaa038f --- /dev/null +++ b/roles/platform/files/aws/plat_aws_authz_roles.tf @@ -0,0 +1,222 @@ +# ------- Cross Account Role ------- +# First create the assume role policy document +data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.xaccount_account_id}:root"] + } + + condition { + test = "StringEquals" + variable = "sts:ExternalId" + + values = [var.xaccount_external_id] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_xaccount_role" { + name = var.xaccount_role_name + description = "CDP Cross Account role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_xaccount_role_policy_doc.json + + tags = merge(var.env_tags,{Name = var.xaccount_role_name}) +} + +# Attach AWS Cross Account Policy to Cross Account Role +resource "aws_iam_role_policy_attachment" "cdp_xaccount_role_attach" { + role = aws_iam_role.cdp_xaccount_role.name + policy_arn = aws_iam_policy.cdp_xaccount_policy.arn +} + +# ------- AWS Service Roles - CDP IDBroker ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_idbroker_role" { + name = var.idbroker_role_name + description = "CDP IDBroker role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_idbroker_role_policy_doc.json + + tags = merge(var.env_tags,{Name = var.idbroker_role_name}) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_idbroker_role_instance_profile" { + name = var.idbroker_role_name + role = aws_iam_role.cdp_idbroker_role.name +} + +# Attach CDP IDBroker Assume Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach1" { + role = aws_iam_role.cdp_idbroker_role.name + policy_arn = aws_iam_policy.cdp_idbroker_policy.arn +} + +# Attach AWS Log Location Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach2" { + + role = aws_iam_role.cdp_idbroker_role.name + policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn +} + + +# ------- AWS Service Roles - CDP Log ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_log_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_log_role" { + name = var.log_role_name + description = "CDP Log role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_log_role_policy_doc.json + + tags = merge(var.env_tags,{Name = var.log_role_name}) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_log_role_instance_profile" { + name = var.log_role_name + role = aws_iam_role.cdp_log_role.name +} + +# Attach AWS Log Location Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" { + + role = aws_iam_role.cdp_log_role.name + policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn +} + +# Attach AWS Bucket Access Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" { + + role = aws_iam_role.cdp_log_role.name + policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn +} + +# ------- AWS Data Access Roles - CDP Datalake Admin ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_datalake_admin_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.caller_account_id}:role/${var.idbroker_role_name}"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_datalake_admin_role" { + name = var.datalake_admin_role_name + description = "CDP Datalake Admin role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc.json + + tags = merge(var.env_tags,{Name = var.datalake_admin_role_name}) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_datalake_admin_role_instance_profile" { + name = var.datalake_admin_role_name + role = aws_iam_role.cdp_datalake_admin_role.name +} + +# Attach AWS Datalake Admin S3 Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_datalake_admin_s3_data_access_policy.arn +} + +# Attach AWS Bucket Access Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn +} + +# ------- AWS Data Access Roles - CDP Ranger Audit ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_ranger_audit_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.caller_account_id}:role/${var.idbroker_role_name}"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_ranger_audit_role" { + name = var.ranger_audit_role_name + description = "CDP Ranger Audit role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc.json + + tags = merge(var.env_tags,{Name = var.ranger_audit_role_name}) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_ranger_audit_role_instance_profile" { + name = var.ranger_audit_role_name + role = aws_iam_role.cdp_ranger_audit_role.name +} + +# Attach AWS Ranger Audit S3 Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn +} + +# Attach AWS Bucket Access Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_bucket_data_access_policy.arn +} diff --git a/roles/platform/template/aws/provider.tf.j2 b/roles/platform/files/aws/provider.tf similarity index 100% rename from roles/platform/template/aws/provider.tf.j2 rename to roles/platform/files/aws/provider.tf diff --git a/roles/platform/files/aws/terraform_variables.tf b/roles/platform/files/aws/terraform_variables.tf new file mode 100644 index 00000000..04a70b77 --- /dev/null +++ b/roles/platform/files/aws/terraform_variables.tf @@ -0,0 +1,147 @@ +# ------- Global settings ------- +variable "aws_profile" { + type = string + description = "Profile for AWS cloud credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +# Region +variable "region" { + type = string + description = "Region which Cloud resources will be created" +} + + +variable "env_tags" { + type = map + description = "Tags applied to provised resources" + + default = { + comment = "Created with Terraform by cloudera-deploy" + } +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "caller_account_id" { + type = string + description = "ID of the Cloud Service Provider Account" +} + +# ------- Policies ------- +# Cross Account Policy (name and document) +variable "xaccount_policy_name" { + type = string + description = "Cross Account Policy name" + +} + +variable "xaccount_account_policy_doc" { + type = string + description = "Location of cross acount policy document" + +} +# CDP IDBroker Assume Role policy +variable "idbroker_policy_name" { + type = string + description = "IDBroker Policy name" + +} + +# CDP Data Access Policies - Log +variable "log_data_access_policy_name" { + type = string + description = "Log Data Access Policy Name" + +} + +variable "log_data_access_policy_doc" { + type = string + description = "Location of Log Data Access Policy" + +} + +# CDP Data Access Policies - ranger_audit_s3 +variable "ranger_audit_s3_policy_name" { + type = string + description = "Ranger S3 Audit Data Access Policy Name" + +} + +variable "ranger_audit_s3_policy_doc" { + type = string + description = "Location of Ranger S3 Audit Data Access Policy" + +} + +# CDP Data Access Policies - datalake_admin_s3 +variable "datalake_admin_s3_policy_name" { + type = string + description = "Datalake Admin S3 Data Access Policy Name" + +} + +variable "datalake_admin_s3_policy_doc" { + type = string + description = "Location of Datalake Admin S3 Data Access Policy" + +} + +# CDP Data Access Policies - bucket_access +variable "bucket_access_policy_name" { + type = string + description = "Bucket Access Data Access Policy Name" + +} + +variable "bucket_access_policy_doc" { + type = string + description = "Bucket Access Data Access Policy" + +} + +# ------- Policies ------- +# Cross Account Role (name and id) +variable "xaccount_role_name" { + type = string + description = "Cross account Assume role Name" +} + +variable "xaccount_account_id" { + type = string + description = "Account ID of the cross account" +} + +variable "xaccount_external_id" { + type = string + description = "External ID of the cross account" +} + +# IDBroker service role +variable "idbroker_role_name" { + type = string + description = "IDBroker service role Name" +} + +# Log service role +variable "log_role_name" { + type = string + description = "Log service role Name" +} + +# CDP Datalake Admin role +variable "datalake_admin_role_name" { + type = string + description = "Datalake Admin role Name" +} + +# CDP Ranger Audit role +variable "ranger_audit_role_name" { + type = string + description = "Ranger Audit role Name" +} diff --git a/roles/platform/tasks/initialize_aws_terraform.yml b/roles/platform/tasks/initialize_aws_terraform.yml index 7dea4e7e..9e726c99 100644 --- a/roles/platform/tasks/initialize_aws_terraform.yml +++ b/roles/platform/tasks/initialize_aws_terraform.yml @@ -42,36 +42,43 @@ src: "{{ __aws_policy_tmpdir.path }}/" dest: "{{ plat__terraform_template_dir }}/plat/policy_docs/" -# Apply template for Terraform provider -- name: Generate Terraform Provider - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/provider.tf.j2' +# Copy Terraform provider file +- name: Copy Terraform Provider file + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/provider.tf' dest: "{{ plat__terraform_template_dir }}/plat/provider.tf" -# Apply template for Terraform backend state -- name: Generate Terraform Backend State - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/backend_state.tf.j2' - dest: "{{ plat__terraform_template_dir }}/plat/backend_state.tf" - -# Apply template for Terraform variables -- name: Generate Terraform Variables - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/terraform_variables.tf.j2' +# Copy Terraform variables file +- name: Copy Terraform Variables declaration file + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/terraform_variables.tf' dest: "{{ plat__terraform_template_dir }}/plat/variables.tf" no_log: false -# Apply template for Terraform auth resources.... +# Copy the Terraform auth resource files.... # ...policies - name: Generate Terraform authz file for policies - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_policies.tf.j2' + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_policies.tf' dest: "{{ plat__terraform_template_dir }}/plat/plat_authz_policies.tf" no_log: false # ...roles - name: Generate Terraform authz file for roles - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_roles.tf.j2' + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/plat_{{ plat__infra_type }}_authz_roles.tf' dest: "{{ plat__terraform_template_dir }}/plat/plat_authz_roles.tf" no_log: false + +# Apply template for Terraform backend state +- name: Generate Terraform Backend State + ansible.builtin.template: + src: 'templates/{{ plat__infra_type }}/backend_state.tf.j2' + dest: "{{ plat__terraform_template_dir }}/plat/backend_state.tf" + +# Create Terraform variable definitions from template +- name: Generate Terraform Variables definition + ansible.builtin.template: + src: 'templates/{{ infra__type }}/terraform.tfvars.j2' + dest: "{{ plat__terraform_template_dir }}/plat/terraform.tfvars" + no_log: false diff --git a/roles/platform/tasks/initialize_teardown_aws_terraform.yml b/roles/platform/tasks/initialize_teardown_aws_terraform.yml index f99c2524..6cbb0edc 100644 --- a/roles/platform/tasks/initialize_teardown_aws_terraform.yml +++ b/roles/platform/tasks/initialize_teardown_aws_terraform.yml @@ -17,25 +17,25 @@ path: "{{ plat__terraform_template_dir }}/plat" state: directory -# Apply template for Terraform provider -- name: Generate Terraform Provider - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/provider.tf.j2' +# Copy Terraform provider +- name: Copy Terraform Provider file + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/provider.tf' dest: "{{ plat__terraform_template_dir }}/plat/provider.tf" +# Copy Terraform variables file +- name: Copy Terraform Variables declaration file + ansible.builtin.copy: + src: 'files/{{ plat__infra_type }}/terraform_variables.tf' + dest: "{{ plat__terraform_template_dir }}/plat/variables.tf" + no_log: false + # Apply template for Terraform backend state - name: Generate Terraform Backend State ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/backend_state.tf.j2' + src: 'templates/{{ plat__infra_type }}/backend_state.tf.j2' dest: "{{ plat__terraform_template_dir }}/plat/backend_state.tf" -# Apply template for Terraform variables -- name: Generate Terraform Variables - ansible.builtin.template: - src: 'template/{{ plat__infra_type }}/terraform_variables.tf.j2' - dest: "{{ plat__terraform_template_dir }}/plat/variables.tf" - no_log: false - - name: Create a temporary directory for policy documents ansible.builtin.tempfile: prefix: "aws-policy-" diff --git a/roles/platform/template/aws/plat_aws_authz_policies.tf.j2 b/roles/platform/template/aws/plat_aws_authz_policies.tf.j2 deleted file mode 100644 index aa4df84f..00000000 --- a/roles/platform/template/aws/plat_aws_authz_policies.tf.j2 +++ /dev/null @@ -1,75 +0,0 @@ -# ------- AWS Cross Account Policy ------- -# The policy here is a dict variable so we'll use the variable -# directly in the aws_iam_policy resource. -resource "aws_iam_policy" "{{ plat__aws_xaccount_policy_name }}" { - name = "{{ plat__aws_xaccount_policy_name }}" - description = "CDP Cross Account policy for {{ plat__namespace }}" - - tags = merge(var.env_tags,{Name = "{{ plat__aws_xaccount_policy_name }}"}) - - policy = <