Add TF example for CDP on AWS

public-cloud/aws/tf/.gitignore

@@ -0,0 +1,35 @@
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible-navigator.log
+runs
+context
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+**/*.tfstate
+**/*.tfstate.*
+
+# Crash log files
+**/crash.log
+**/crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+**/*.tfvars
+**/*.tfvars.json

public-cloud/aws/tf/README.md

@@ -0,0 +1,71 @@
+# CDP Public Cloud - Environment and Datalake Base Example
+
+> Constructs a CDP Public Cloud Environment and Datalake. Uses the [terraform-cdp-modules](https://github.com/cloudera-labs/terraform-cdp-modules), called via Ansible, to generate the AWS infrastructure pre-requisite resources and the CDP artifacts.
+
+> **NOTE:** This deployment example does not use a `definition.yml` based configuration file. Instead a standard Ansible extra vars configuration file is used.
+
+## Requirements
+
+To run, you need:
+
+* Docker (or a Docker clone[^1])
+* AWS credentials (set via `AWS_PROFILE`)
+* CDP credentials (set via `CDP_PROFILE`)
+
+[^1]: For example, [OrbStack](https://orbstack.dev) works well on OSX.
+
+## Set Up
+
+First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the top-level [README](../../../README.md#setting-up-ansible-navigator).
+
+Then, clone this project and change your working directory.
+
+```bash
+git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/aws/base
+```
+
+## Configure
+
+Set the required environment variables:
+
+```bash
+export AWS_PROFILE=your-aws-profile
+export CDP_PROFILE=your-cdp-profile
+```
+
+Tweak the `config.yml` parameters to your liking. Notably, you should add and/or change:
+
+```yaml
+name_prefix:    ex01      # Keep this short (4-7 characters)
+infra_region:   us-east-2 # CSP region for infra
+
+deployment_template: public # Specify the deployment pattern below. Options are public, semi-private or private
+```
+
+NOTE: You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e name_prefix=ex01`. See the [FAQ](../../../FAQ.md#how-to-i-add-extra-variables-and-tags-to-ansible-navigator) for details.
+
+### SSH Keys
+
+This definition will create a new SSH keypair on the host of the name `<name_prefix>-ssh-key.{pem,pub}`. This is stored in the `./pbc_infra_tf` directory. A AWS Keypair will be created using the generated public key.
+
+## Execute
+
+Then set up the CDP Public Cloud by running the playbook:
+
+```bash
+ansible-navigator run main.yml
+```
+
+### Terraform resource files 
+
+The Terraform root module resource files run by the playbook are in the `./pbc_infra_tf/` (for cloud infrastructure deployment) `./pbc_deploy_tf/` (CDP deployment) sub-directories. 
+
+Standard Terraform commands - e.g. `terraform output`, `terraform console`, can be run from within these directories.
+
+## Tear Down
+
+Tear down the CDP Public Cloud by running the playbook:
+
+```bash
+ansible-navigator run teardown.yml
+```

public-cloud/aws/tf/ansible-navigator.yml

@@ -0,0 +1,65 @@
+---
+
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible-navigator:
+  playbook-artifact:
+      save-as: "runs/{playbook_name}-{time_stamp}.json"
+
+  ansible-runner:
+    artifact-dir: runs
+    rotate-artifacts-count: 3
+
+  logging:
+    level: debug
+    append: False
+
+  ansible:
+    inventory:
+      entries:
+        - inventory.ini
+
+  execution-environment:
+    container-engine: docker
+    enabled: True
+    environment-variables:
+      pass:
+        - AWS_PROFILE
+        - CDP_PROFILE
+      set:
+        ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks"
+        ANSIBLE_GATHERING: "smart"
+        ANSIBLE_DEPRECATION_WARNINGS: False
+        ANSIBLE_HOST_KEY_CHECKING: False
+        ANSIBLE_SSH_RETRIES: 10
+    image: ghcr.io/cloudera-labs/cldr-runner:aws-devel
+    pull:
+      arguments:
+        - "--tls-verify=false"
+    volume-mounts:
+      - src: "${ANSIBLE_COLLECTIONS_PATH}"
+        dest: "${ANSIBLE_COLLECTIONS_PATH}"
+        options: "Z"
+      - src: "~/.aws"
+        dest: "/runner/.aws"
+        options: "Z"
+      - src: "~/.cdp"
+        dest: "/runner/.cdp"
+        options: "Z"
+      - src: "~/.ssh"
+        dest: "/runner/.ssh"
+        options: "Z"
+    container-options:
+      - "--network=host"

public-cloud/aws/tf/config.yml

@@ -0,0 +1,13 @@
+---
+name_prefix: "<ENTER_VALUE>" # You must specify a name prefix
+
+infra_region: eu-west-1 # CSP region for infra
+
+deployment_template: semi-private  # Specify the deployment pattern below. Options are public, semi-private or private
+
+# Limit to the caller/controller
+allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}"
+
+ingress_extra_cidrs_and_ports:
+    cidrs: "{{ allowed_cidrs }}"
+    ports: [22, 443]

public-cloud/aws/tf/inventory.ini

@@ -0,0 +1,15 @@
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}"

public-cloud/aws/tf/main.yml

@@ -0,0 +1,70 @@
+---
+
+# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Set up the cloudera-deploy variables
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: Set up CDP Public Cloud infrastructure
+      block:
+        - name: Create TF input variables file for infra pre-reqs module
+          ansible.builtin.template:
+            src: cdp_aws_prereqs.tfvars.j2
+            dest: "pbc_infra_tf/terraform.tfvars"
+
+        - name: Terraform apply to set up the Cloud infrastructure
+          # cloud.terraform.terraform: # NOTE: When use this get error similar to https://github.com/ansible-collections/cloud.terraform/issues/86
+          community.general.terraform:
+            project_path: "pbc_infra_tf/"
+            state: "present"
+            force_init: true
+            # provider_upgrade: true
+
+    - name: Set up CDP Public Cloud (Env and DL example)
+      block:
+        - name: Create TF input variables file for CDP deploy module
+          ansible.builtin.template:
+            src: cdp_aws_deploy.tfvars.j2
+            dest: "pbc_deploy_tf/terraform.tfvars"
+          vars:
+            infra_type: "aws"
+            aws_key_pair: "{{ lookup('cloud.terraform.tf_output', 'aws_key_name', project_path='pbc_infra_tf/') }}"
+            aws_vpc_id: "{{ lookup('cloud.terraform.tf_output', 'aws_vpc_id', project_path='pbc_infra_tf/') }}"
+            aws_public_subnet_ids: "{{ lookup('cloud.terraform.tf_output', 'aws_public_subnet_ids', project_path='pbc_infra_tf/') }}"
+            aws_private_subnet_ids: "{{ lookup('cloud.terraform.tf_output', 'aws_private_subnet_ids', project_path='pbc_infra_tf/') }}"
+
+            aws_security_group_default_id: "{{ lookup('cloud.terraform.tf_output', 'aws_security_group_default_id', project_path='pbc_infra_tf/') }}"
+            aws_security_group_knox_id: "{{ lookup('cloud.terraform.tf_output', 'aws_security_group_knox_id', project_path='pbc_infra_tf/') }}"
+
+            data_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_data_storage_location', project_path='pbc_infra_tf/') }}"
+            log_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_log_storage_location', project_path='pbc_infra_tf/') }}"
+            backup_storage_location: "{{ lookup('cloud.terraform.tf_output', 'aws_backup_storage_location', project_path='pbc_infra_tf/') }}"
+
+            aws_xaccount_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_xaccount_role_arn', project_path='pbc_infra_tf/') }}"
+            aws_datalake_admin_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_datalake_admin_role_arn', project_path='pbc_infra_tf/') }}"
+            aws_ranger_audit_role_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_ranger_audit_role_arn', project_path='pbc_infra_tf/') }}"
+
+            aws_log_instance_profile_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_log_instance_profile_arn', project_path='pbc_infra_tf/') }}"
+            aws_idbroker_instance_profile_arn: "{{ lookup('cloud.terraform.tf_output', 'aws_idbroker_instance_profile_arn', project_path='pbc_infra_tf/') }}"
+
+        - name: Terraform apply to set up CDP (may take up to 1 hour to complete)
+          # cloud.terraform.terraform: # NOTE: When use this get error similar to https://github.com/ansible-collections/cloud.terraform/issues/86
+          community.general.terraform:
+            project_path: "pbc_deploy_tf/"
+            state: "present"
+            force_init: true
+            # provider_upgrade: true

public-cloud/aws/tf/pbc_deploy_tf/main.tf

@@ -0,0 +1,28 @@
+module "cdp_deploy" {
+  source = "git::https://github.com/cloudera-labs/terraform-cdp-modules.git//modules/terraform-cdp-deploy?ref=v0.2.0"
+
+  env_prefix          = var.env_prefix
+  infra_type          = "aws"
+  region              = var.aws_region
+  keypair_name        = var.aws_key_pair
+  deployment_template = var.deployment_template
+
+  aws_vpc_id             = var.aws_vpc_id
+  aws_public_subnet_ids  = var.aws_public_subnet_ids
+  aws_private_subnet_ids = var.aws_private_subnet_ids
+
+  aws_security_group_default_id = var.aws_security_group_default_id
+  aws_security_group_knox_id    = var.aws_security_group_knox_id
+
+  data_storage_location   = var.data_storage_location
+  log_storage_location    = var.log_storage_location
+  backup_storage_location = var.backup_storage_location
+
+  aws_xaccount_role_arn       = var.aws_xaccount_role_arn
+  aws_datalake_admin_role_arn = var.aws_datalake_admin_role_arn
+  aws_ranger_audit_role_arn   = var.aws_ranger_audit_role_arn
+
+  aws_log_instance_profile_arn      = var.aws_log_instance_profile_arn
+  aws_idbroker_instance_profile_arn = var.aws_idbroker_instance_profile_arn
+
+}

public-cloud/aws/tf/pbc_deploy_tf/outputs.tf

@@ -0,0 +1,7 @@
+output "cdp_environment_name" {
+  value = module.cdp_deploy.cdp_environment_name
+}
+
+output "cdp_environment_crn" {
+  value = module.cdp_deploy.cdp_environment_crn
+}