From 36c93302e33609bce706bc31b3f7f6a7e572ce84 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:30:58 -0500 Subject: [PATCH 01/45] Initial commit of PvC on AWS Signed-off-by: Jim Enright --- private-cloud/ecs-on-aws/.gitignore | 17 ++ private-cloud/ecs-on-aws/README.md | 195 +++++++++++++ .../ecs-on-aws/ansible-navigator.yml | 50 ++++ private-cloud/ecs-on-aws/base_postfix.yml | 38 +++ private-cloud/ecs-on-aws/base_setup.yml | 38 +++ private-cloud/ecs-on-aws/cluster.yml | 275 ++++++++++++++++++ private-cloud/ecs-on-aws/config.yml | 18 ++ private-cloud/ecs-on-aws/definition.yml | 248 ++++++++++++++++ .../ecs-on-aws/execution-environment.yml | 18 ++ private-cloud/ecs-on-aws/external_setup.yml | 48 +++ private-cloud/ecs-on-aws/group_vars/all.yml | 3 + private-cloud/ecs-on-aws/hostvars.j2 | 9 + private-cloud/ecs-on-aws/instance_vars.j2 | 13 + private-cloud/ecs-on-aws/internal_setup.yml | 38 +++ private-cloud/ecs-on-aws/pre_setup.yml | 231 +++++++++++++++ .../ecs-on-aws/pre_setup_resources.yml | 16 + private-cloud/ecs-on-aws/pre_teardown.yml | 24 ++ .../ecs-on-aws/validate_dns_lookups.yml | 27 ++ 18 files changed, 1306 insertions(+) create mode 100644 private-cloud/ecs-on-aws/.gitignore create mode 100644 private-cloud/ecs-on-aws/README.md create mode 100644 private-cloud/ecs-on-aws/ansible-navigator.yml create mode 100644 private-cloud/ecs-on-aws/base_postfix.yml create mode 100644 private-cloud/ecs-on-aws/base_setup.yml create mode 100644 private-cloud/ecs-on-aws/cluster.yml create mode 100644 private-cloud/ecs-on-aws/config.yml create mode 100644 private-cloud/ecs-on-aws/definition.yml create mode 100644 private-cloud/ecs-on-aws/execution-environment.yml create mode 100644 private-cloud/ecs-on-aws/external_setup.yml create mode 100644 private-cloud/ecs-on-aws/group_vars/all.yml create mode 100644 private-cloud/ecs-on-aws/hostvars.j2 create mode 100644 private-cloud/ecs-on-aws/instance_vars.j2 create mode 100644 private-cloud/ecs-on-aws/internal_setup.yml create mode 100644 private-cloud/ecs-on-aws/pre_setup.yml create mode 100644 private-cloud/ecs-on-aws/pre_setup_resources.yml create mode 100644 private-cloud/ecs-on-aws/pre_teardown.yml create mode 100644 private-cloud/ecs-on-aws/validate_dns_lookups.yml diff --git a/private-cloud/ecs-on-aws/.gitignore b/private-cloud/ecs-on-aws/.gitignore new file mode 100644 index 0000000..e5a1363 --- /dev/null +++ b/private-cloud/ecs-on-aws/.gitignore @@ -0,0 +1,17 @@ +# Ansible Navigator assets +ansible-navigator.log +runs +context + +# Terraform deployments +tf_deployment* + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Static inventory files +inventory_static* \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md new file mode 100644 index 0000000..d5d6d04 --- /dev/null +++ b/private-cloud/ecs-on-aws/README.md @@ -0,0 +1,195 @@ +# PvC ECS Cluster on AWS + +> Constructs a CDP Private Cloud Base cluster suitable for enablement in configuring and using Ozone on a Private Cloud ECS Data Services (DS) cluster. + +## Requirements + +This example requires an execution environment with dependencies to run the automation; and a set of configuration variables. + +We provide instructions for using a container based execution environment. + +### Container Execution Environment + +1. Create and activate a new `virtualenv` and install `ansible-core` and `ansible-navigator` + + ```bash + python -m venv ~/cdp-navigator; + + source ~/cdp-navigator/bin/activate; + + pip install ansible-core==2.12.10 ansible-navigator==3.4.0 + ``` + +1. Clone this repository. + + ```bash + git clone https://github.com/cloudera-labs/cloudera-deploy.git; + ``` + +1. Change your working directory to this project. + + ```bash + cd cloudera-deploy/private-cloud/ecs-on-aws + ``` + +1. We currently need to build a local `cldr-runner` image for use as an Execution Environment. _(This is necessary at this point in the release cycle. It will become optional.)_ + + ```bash + ansible-navigator builder build --prune-images -v 3 --tag ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + ``` + +This step sometimes takes a ~~number of minutes~~ long time to complete. YMMV. + +Once constructed, you can check the image by running `ansible-navigator` and using the prompts to examine: + +```bash +ansible-navigator images +``` + +Or by running the Docker command directly, `docker image ls`. + +### Configuration Variables + +Configuration is passed via environment variables and an user-facing configuration file. + +#### Environment Variables + +* Set up the following definition environment variables: + + | Variable | Description | Status | + |----------|-------------|--------| + | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/wmudge/.ssh/demo_ops.pub` | Mandatory | + | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/wmudge/Documents/webster_mudge_2022_2023_cloudera_license.txt` | Mandatory | + | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | + | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | + | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. Used also for remote storage of Terraform state in AWS. | Mandatory | + +> **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. + + +#### Configuration file variables + +Edit the `config.yml` user-facing configuration file to match your particular deployment. + +*NOTE:* `name_prefix` should be 4-8 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. + +```yaml +name_prefix: "labaw" # CHANGE THIS +owner_prefix: "pvc-ecs" +owner_email: "example@cloudera.com" +infra_region: "eu-west-1" +infra_type: "aws" # "aws", "static" +domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) +realm: "CLDR.EXAMPLE" # The Kerberos realm +common_password: "Example776" +admin_password: "Example776" +deployment_tags: + owner: "{{ owner_prefix }}" + email: "{{ owner_email }}" + project: "PvC ECS Lab - {{ owner_prefix }}-{{ name_prefix }}" + enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy +``` + +## Execution + +### Pre-setup Playbook + +This definition-specific playbook includes tasks such as: +* Instructure provisioning +* FreeIPA DNS and KRB services provisioning + +Run the following command + +```bash +ansible-navigator run pre_setup.yml \ +-e @config.yml \ +-e @definition.yml +``` + +Once the pre-setup playbook completes confirm that: + +* You can connect to each node via the inventory - see Confirm SSH Connectivity. Note that a A `validate_dns_lookups.yml` Playbook exists to check connectivity. +* Connect to FreeIPA UI and login with the `IPA_USER` and `IPA_PASSWORD` credentials in the configuration file. See Cluster Access for more details. + +### Platform Playbooks + +These playbooks configure and deploy PVC Base and, optionally, ECS. They use the infrastructure provisioned (or assigned, if using `static` inventory). + +Tasks include: +* System/host configuration +* Cloudera Manager server and agent installation and configuration +* Cluster template imports + +Run the following: + +```bash +# Run the 'external' system configuration +ansible-navigator run external_setup.yml \ + -e @lab_configs/config.yml \ + -i inventory_static__aws.ini +``` + +```bash +# Run the 'internal' Cloudera installations and configurations +ansible-navigator run internal_setup.yml \ + -e @lab_configs/config.yml \ + -i inventory_static__aws.ini +``` + +```bash +# Run the Cloudera cluster configuration and imports +ansible-navigator run base_setup.yml \ + -e @lab_configs/config.yml \ + -i inventory_static__aws.ini +``` + +And lastly, the _postfix_: + +```bash +ansible-navigator run base_postfix.yml \ + -e @lab_configs/config.yml \ + -i inventory_static__aws.ini +``` + +## Cluster Access + +Once the cluster is up, you can access all of the UIs within, including the FreeIPA sidecar, via a SSH tunnel: + +```bash +ssh -D -q -C -N @ +``` + +and use a SOCKS5 proxy switcher in your browser. You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. + +In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! + +```bash +ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @ +``` + +## Teardown + +Run the following: + +```bash +ansible-navigator run pre_teardown.yml \ + -e @lab_configs/config.yml \ + -e @definition.yml \ + -i inventory_static__aws.ini +``` + +You can also run `terraform destroy` within the `tf_deployment_*` directory. + +## Troubleshooting + +### Confirm SSH Connectivity + +Run the following from _the root of the definition project, i.e. `demo-definitions`_: + +```bash +ansible -m ansible.builtin.ping -i inventory_static__aws.ini all +``` + +This will check to see if the inventory file is well constructed, etc. diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml new file mode 100644 index 0000000..d60d305 --- /dev/null +++ b/private-cloud/ecs-on-aws/ansible-navigator.yml @@ -0,0 +1,50 @@ +--- + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + - SSH_PUBLIC_KEY_FILE + - CDP_LICENSE_FILE + - IPA_USER + - IPA_PASSWORD + set: + ANSIBLE_SSH_CONTROL_PATH: "/dev/shm/cp%%h-%%p-%%r" + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${SSH_PUBLIC_KEY_FILE}" + dest: "${SSH_PUBLIC_KEY_FILE}" + - src: "${CDP_LICENSE_FILE}" + dest: "${CDP_LICENSE_FILE}" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/private-cloud/ecs-on-aws/base_postfix.yml b/private-cloud/ecs-on-aws/base_postfix.yml new file mode 100644 index 0000000..c7336f5 --- /dev/null +++ b/private-cloud/ecs-on-aws/base_postfix.yml @@ -0,0 +1,38 @@ +--- + +- name: Postfix CDP Private Cloud clusters + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Postfix clusters for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_postfix.yml diff --git a/private-cloud/ecs-on-aws/base_setup.yml b/private-cloud/ecs-on-aws/base_setup.yml new file mode 100644 index 0000000..be6790b --- /dev/null +++ b/private-cloud/ecs-on-aws/base_setup.yml @@ -0,0 +1,38 @@ +--- + +- name: Set up CDP Private Cloud clusters + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up clusters for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_setup.yml diff --git a/private-cloud/ecs-on-aws/cluster.yml b/private-cloud/ecs-on-aws/cluster.yml new file mode 100644 index 0000000..85cdecb --- /dev/null +++ b/private-cloud/ecs-on-aws/cluster.yml @@ -0,0 +1,275 @@ +--- + +cluster_name_base: PVC-Base + +clusters: + - name: PVC-ECS + type: ecs + pvc_repository: "{{ ecs_options.repository }}" + application_domain: "{{ name_prefix }}.{{ domain }}" + datalake: "{{ cluster_name_base }}" + ecs_database_mode: embedded + services: [ DOCKER, ECS ] + security: + tls: true + repositories: "{{ ecs_options.parcel_repositories }}" + configs: + DOCKER: + SERVICEWIDE: + defaultDataPath: "/mnt/docker" + docker_images_destination_registry_user: registry-user + ECS: + SERVICEWIDE: + docker: docker + cp_prometheus_ingress_user: cloudera-manager + cp_prometheus_ingress_password: "{{ admin_password }}" + infra_prometheus_ingress_user: cloudera-manager + infra_prometheus_ingress_password: "{{ admin_password }}" + longhorn_replication: 2 + lsoDataPath: "/ecs/local" + defaultDataPath: "/ecs/longhorn-storage" + nfs_provisioned: 800 + nfs_over_provisioning: 800 + # TODO: Create cert and key for ECS ingress controller + # ssl_certificate: /opt/cloudera/security/pki/ecs.pem + # ssl_private_key: /opt/cloudera/security/pki/ecs.key + host_templates: + ECSMaster: + DOCKER: [ DOCKER_SERVER ] + ECS: [ ECS_SERVER ] + ECSWorker: + DOCKER: [ DOCKER_SERVER ] + ECS: [ ECS_AGENT ] + controlplane_config: + ContainerInfo: + Mode: public + CopyDocker: false + Database: + Mode: embedded + EmbeddedDbStorage: 50 + + - name: PVC-Base + type: base + services: [ATLAS, HBASE, HDFS, HIVE, HIVE_ON_TEZ, INFRA_SOLR, KAFKA, OZONE, RANGER, SPARK_ON_YARN, TEZ, YARN, ZOOKEEPER] + security: + kerberos: true + tls: true + repositories: "{{ repositories }}" + configs: + ATLAS: + SERVICEWIDE: + kerberos.auth.enable: true + ATLAS_SERVER: + atlas_authentication_method_file: true + atlas_admin_password: "{{ common_password }}" + atlas_sso_knox_enabled: false + atlas_authentication_method_trustedproxy: false + atlas_authentication_method_pam: false + atlas_authentication_method_ldap: true + atlas_authentication_method_ldap_url: "{{ auth_provider.ldap_url }}" + atlas_authentication_method_ldap_bind_dn: "{{ auth_provider.ldap_bind_user_dn }}" + atlas_authentication_method_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" + atlas_authentication_method_ldap_userDNpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" + atlas_authentication_method_ldap_base_dn: "{{ auth_provider.ldap_search_base.user }}" + atlas_authentication_method_ldap_user_searchfilter: "{{ auth_provider.ldap_search_filter.user }}" + atlas_authentication_method_ldap_groupSearchBase: "{{ auth_provider.ldap_search_base.group }}" + atlas_authentication_method_ldap_groupSearchFilter: "{{ auth_provider.ldap_search_filter.member }}" + atlas_authentication_method_ldap_groupRoleAttribute: "{{ auth_provider.ldap_attr_group_name }}" + atlas_authentication_method_ldap_type: "ldap" + atlas_authentication_method_ldap_default_role: "ROLE_ADMIN" + HBASE: + SERVICEWIDE: + hadoop_secure_web_ui: true + hbase_graceful_stop_timeout: 5 + REGIONSERVER: + hbase_regionserver_java_heapsize: 268435456 + MASTER: + hbase_master_java_heapsize: 134217728 + GATEWAY: + hbase_client_java_heapsize: 134217728 + # NOTE: Have removed /mnt from the various HFDS directories + HDFS: + SERVICEWIDE: + hdfs_verify_ec_with_topology_enabled: false + enable_ranger_authorization: true + hadoop_secure_web_ui: false + trusted_realms: "{{ realm | upper }}, {{ realm | lower }}" + DATANODE: + datanode_java_heapsize: 134217728 + dfs_data_dir_list: /dfs/dn + dfs_datanode_max_locked_memory: 0 + dfs_datanode_failed_volumes_tolerated: 0 + GATEWAY: + hdfs_client_java_heapsize: 134217728 + NAMENODE: + dfs_name_dir_list: /dfs/nn + namenode_java_heapsize: 134217728 + role_config_suppression_namenode_java_heapsize_minimum_validator: true + SECONDARYNAMENODE: + secondary_namenode_java_heapsize: 134217728 + fs_checkpoint_dir_list: /dfs/snn + HIVE: + SERVICEWIDE: + ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" + GATEWAY: + hive_client_java_heapsize: 134217728 + HIVEMETASTORE: + hive_metastore_java_heapsize: 268435456 + metastore_canary_health_enabled: false + HIVE_ON_TEZ: + SERVICEWIDE: + ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" + KAFKA: + SERVICEWIDE: + kerberos.auth.enable: true + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + producer.metrics.enable: false + service_config_suppression_kafka_broker_count_validator: true + service_config_suppression_offsets.topic.replication.factor: true + service_config_suppression_transaction.state.log.replication.factor: true + KAFKA_BROKER: + graceful_stop_timeout: 5 + OZONE: + SERVICEWIDE: + ozone.security.enabled: true + ozone.service.id: ozone1 + ozone.security.http.kerberos.enabled: false + service_config_suppression_ozone_manager_count_validator: true + service_config_suppression_storage_container_manager_count_validator: true + OZONE_MANAGER: + om_max_heap_size: 2048 + STORAGE_CONTAINER_MANAGER: + scm_max_heap_size: 2048 + OZONE_DATANODE: + ozone_datanode_heap_size: 2048 + OZONE_PROMETHEUS: + ozone.prometheus.http-port: 19090 + RANGER: + SERVICEWIDE: + keyadmin_user_password: "{{ common_password }}" + rangeradmin_user_password: "{{ common_password }}" + rangertagsync_user_password: "{{ common_password }}" + rangerusersync_user_password: "{{ common_password }}" + RANGER_ADMIN: + ranger_admin_max_heap_size: 1024 + ranger_authentication_method: "LDAP" + ranger.ldap.url: "{{ auth_provider.ldap_url }}" + ranger.ldap.bind.dn: "{{ auth_provider.ldap_bind_user_dn }}" + ranger_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" + ranger.ldap.user.dnpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" + ranger.ldap.base.dn: "{{ auth_provider.ldap_base_dn }}" + ranger.ldap.user.searchfilter: "(&(sAMAccountName={0})(objectClass=person))" + ranger.ldap.group.searchfilter: "(&(member={0})(objectClass=group))" + ranger.ldap.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" + ranger.ldap.group.roleattribute: "{{ auth_provider.ldap_attr_group_name }}" + ranger.ldap.referral: "follow" + RANGER_TAGSYNC: + ranger_tagsync_max_heap_size: 1024 + RANGER_USERSYNC: + ranger_usersync_max_heap_size: 1024 + ranger.usersync.ldap.binddn: "{{ auth_provider.ldap_bind_user_dn }}" + ranger.usersync.ldap.url: "{{ auth_provider.ldap_url }}" + ranger_usersync_ldap_ldapbindpassword: "{{ auth_provider.ldap_bind_password }}" + ranger.usersync.ldap.referral: follow + ranger.usersync.ldap.username.caseconversion: lower + ranger.usersync.ldap.groupname.caseconversion: lower + ranger.usersync.ldap.grouphierarchylevels: 0 + ranger.usersync.group.based.role.assignment.rules: '&ROLE_SYS_ADMIN:g:field-se' + ranger.usersync.group.memberattributename: "{{ auth_provider.ldap_attr_group_membership }}" + ranger.usersync.group.nameattribute: "{{ auth_provider.ldap_attr_group_name }}" + ranger.usersync.group.objectclass: group + ranger.usersync.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" +# ranger.usersync.group.searchfilter: "" +# ranger.usersync.ldap.user.searchfilter: "" + ranger.usersync.ldap.user.nameattribute: "{{ auth_provider.ldap_attr_user_name }}" + ranger.usersync.ldap.user.objectclass: person + ranger.usersync.ldap.user.searchbase: "{{ auth_provider.ldap_search_base.user }}" + ranger.usersync.source.impl.class: org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder + ranger.usersync.user.searchenabled: 'true' + +#obsolete in 7.1.6 ranger.usersync.ldap.searchBase: "{{ auth_provider.ldap_base_dn }}" +#obsolete in 7.1.6 ranger.usersync.ldap.user.groupnameattribute: "{{ auth_provider.ldap_attr_group_name }}" +#obsolete in 7.1.6 ranger.usersync.group.search.first.enabled: 'true' +#obsolete in 7.1.6 ranger.usersync.group.searchenabled: 'true' + INFRA_SOLR: + SERVICEWIDE: + #set false for 7.1.7 since it doesnt build cm_solr + enable_ranger_authorization: false + SOLR_SERVER: + solr_graceful_stop_timeout: 5 + SPARK_ON_YARN: + SPARK_YARN_HISTORY_SERVER: + ssl_enabled: true + history_server_spnego_enabled: false + YARN: + SERVICEWIDE: + hadoop_secure_web_ui: false + JOBHISTORY: + mr2_jobhistory_java_heapsize: 134217728 + NODEMANAGER: + node_manager_java_heapsize: 134217728 + yarn_nodemanager_resource_memory_mb: 2048 + yarn_nodemanager_resource_cpu_vcores: 2 + RESOURCEMANAGER: + resource_manager_java_heapsize: 134217728 + yarn_scheduler_maximum_allocation_mb: 2048 + yarn_scheduler_maximum_allocation_vcores: 2 + ZOOKEEPER: + SERVICEWIDE: + zookeeper_datadir_autocreate: true + service_config_suppression_server_count_validator: true + SERVER: + zookeeper_server_java_heapsize: 134217728 + host_templates: + Masters: + ATLAS: [ATLAS_SERVER] + HBASE: [MASTER, REGIONSERVER, GATEWAY] + HDFS: [NAMENODE, SECONDARYNAMENODE, DATANODE] + HIVE: [HIVEMETASTORE, GATEWAY] + HIVE_ON_TEZ: [HIVESERVER2, GATEWAY] + INFRA_SOLR: [SOLR_SERVER] + KAFKA: [KAFKA_BROKER] + OZONE: [GATEWAY, OZONE_MANAGER, OZONE_RECON, STORAGE_CONTAINER_MANAGER, OZONE_DATANODE, S3_GATEWAY] + RANGER: [RANGER_ADMIN, RANGER_TAGSYNC, RANGER_USERSYNC] + SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER, GATEWAY] + TEZ: [GATEWAY] + YARN: [RESOURCEMANAGER, NODEMANAGER, JOBHISTORY, GATEWAY] + ZOOKEEPER: [SERVER] + + Workers: + HBASE: [REGIONSERVER, GATEWAY] + HDFS: [DATANODE] + HIVE: [GATEWAY] + HIVE_ON_TEZ: [GATEWAY] + OZONE: [OZONE_DATANODE, GATEWAY] + TEZ: [GATEWAY] + SPARK_ON_YARN: [GATEWAY] + YARN: [NODEMANAGER, GATEWAY] + ZOOKEEPER: [SERVER] + +mgmt: + name: Cloudera Management Service + services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] + security: + tls: true + configs: + HOSTMONITOR: + firehose_heapsize: 1073741824 + firehose_non_java_memory_bytes: 2147483648 + SERVICEMONITOR: + firehose_heapsize: 2147483648 + firehose_non_java_memory_bytes: 6442450944 + role_config_suppression_firehose_service_monitor_non_java_memory_role_validator: true + +hosts: + configs: + host_default_proc_memswap_thresholds: + warning: never + critical: never + host_memswap_thresholds: + warning: never + critical: never + host_config_suppression_agent_system_user_group_validator: true + host_config_suppression_memory_overcommitted_validator: true diff --git a/private-cloud/ecs-on-aws/config.yml b/private-cloud/ecs-on-aws/config.yml new file mode 100644 index 0000000..40ed695 --- /dev/null +++ b/private-cloud/ecs-on-aws/config.yml @@ -0,0 +1,18 @@ +--- + +name_prefix: "labaw" # CHANGE THIS +owner_prefix: "pvc-ecs" +owner_email: "example@cloudera.com" +infra_region: "eu-west-1" +infra_type: "aws" # "aws", "static" +domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) +realm: "CLDR.EXAMPLE" # The Kerberos realm +common_password: "Example776" +admin_password: "Example776" # "Main" default password +deployment_tags: + owner: "{{ owner_prefix }}" + email: "{{ owner_email }}" + project: "PvC ECS Lab - {{ owner_prefix }}-{{ name_prefix }}" + enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy diff --git a/private-cloud/ecs-on-aws/definition.yml b/private-cloud/ecs-on-aws/definition.yml new file mode 100644 index 0000000..17d9a04 --- /dev/null +++ b/private-cloud/ecs-on-aws/definition.yml @@ -0,0 +1,248 @@ +--- + +kerberos_activated: True +encryption_activated: True +# tls_activated: True +# autotls: False + +repositories: + # 7.1.8 CHF4 + - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/ +cloudera_manager_version: 7.10.1 + +jdk_version: 11 + +# ECS details +ecs_options: + # Version 1.5.1 + repository: https://archive.cloudera.com/p/cdp-pvc-ds/1.5.1/ + parcel_repositories: + - https://archive.cloudera.com/p/cdp-pvc-ds/1.5.1/parcels/ + +# SSH +public_key_id: "{{ owner_prefix }}-{{ name_prefix }}" +public_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" +public_key_text: "{{ lookup('ansible.builtin.file', lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE')) }}" + +# Cloudera license +license_file: "{{ lookup('ansible.builtin.env', 'CDP_LICENSE_FILE') }}" + +# Terraform settings +terraform: + state: + storage: local + create_remote_storage: False + s3_region: "{{ infra_region }}" + s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" + +# DNS +dns_provider: "freeipa" + +# Connect FreeIPA to Knox and Ranger +freeipa_activated: yes + +# FreeIPA realm settings +freeipa: + realm: "{{ realm }}" + +##### + +# Vars from legacy cloudera-Deploy +use_auto_repo_mirror: no +use_default_cluster_definition: no +use_download_mirror: no +preload_cm_parcel_repo: yes +teardown_everything: yes + +# Defer user and group creation to SSSD/FreeIPA registration +#Use when you dont need ansible to run "ca_server" +skip_user_group_init: no + +# Cloudera Manager details + +cloudera_manager_options: + CUSTOM_BANNER_HTML: "1.5.1 - PvC ECS Lab ({{ name_prefix }})" + SESSION_TIMEOUT: 43200 + PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 + # NOTE: Disabled in ChuckL's latest definition + # PHONE_HOME: false + # PROXYUSER_KNOX_GROUPS: "*" + # PROXYUSER_KNOX_HOSTS: "*" + # PROXYUSER_KNOX_PRINCIPAL: "knox" + # PROXYUSER_KNOX_USERS: "*" + KRB_AUTH_ENABLE: "true" + +# License options (this is due to a hardcoded tmp directory on the target/manager node) +license_local_tmp_path: /tmp/cloudera_license.txt + +############### +### RDBMS ### +############### +database_type: postgresql +database_version: 10 +database_tls: true + +#########++++ + +## Red Hat FreeIPA +krb5_kdc_type: Red Hat IPA +krb5_realm: "{{ realm }}" +krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user | default(lookup('env', 'IPA_USER', default='Undefined')) }}@{{ krb5_realm }}" +krb5_kdc_admin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" +krb5_enc_types: "aes256-cts aes128-cts" +# TODO: This needs to either be explicit or rendered from a host with DNS resolution +# Cannot rely on localhost to have DNS access, e.g. private networking on EC2 +#krb5_kdc_host: "{{ lookup('community.general.dig', '_ldap._tcp.' + domain +'./SRV', 'flat=0', wantlist=True, fail_on_error=True) | map(attribute='target') | first }}" +krb5_kdc_host: "{{ groups['freeipa'] | first | default('') }}" + +############################################ +### CM External Auth - FreeIPA as LDAP ### +############################################ + +# Settings for FreeIPA sidecar deployment +cloudera_manager_external_auth: + provider: freeipa + external_first: no + external_only: no + +auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" + +base_dn: "dc={{ (krb5_realm | lower).split('.') | join(',dc=') }}" +user_dn: "cn=users,cn=accounts,{{ base_dn }}" +group_dn: "cn=groups,cn=accounts,{{ base_dn }}" + +auth_providers: + freeipa: + ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}:636" + ldap_base_dn: "{{ base_dn }}" + ldap_bind_user_dn: "uid=admin,{{ user_dn }}" + ldap_bind_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + ldap_search_base: + user: "{{ user_dn }}" + group: "{{ group_dn }}" + ldap_object_class: + user: "person" + group: "groupofnames" + ldap_search_filter: + user: uid={0} + member: (member={0}) + ldap_attr_user_name: "uid" + ldap_attr_group_name: "cn" + ldap_attr_group_membership: "member" + type: LDAP + +## TLS signing +skip_ipa_signing: no # This is the default +remote_ipa_server: "{{ groups['freeipa'] | first | default('') }}" + +tls_key_password: "{{ common_password }}" +tls_keystore_password: "{{ common_password }}" +tls_truststore_password: "{{ common_password }}" + +ca_server_attrs_root: + CN: 'SE-Root CA' +ca_server_attrs_intermediate: + CN: 'SE-Intermediate CA' + +# Infrastructure +infra: + aws: + region: "{{ infra_region }}" + enable_dns: no + public_subnets: # TODO: Implement better selection of AZs + - az: "{{ infra_region }}a" + private_subnets: + - az: "{{ infra_region }}b" + - az: "{{ infra_region }}c" + default_security_group: + ingress: + - cidr: [ "0.0.0.0/0" ] + from: 22 + to: 22 + protocol: TCP + desc: SSH (Ansible) + nodes: + # Sidecar FreeIPA for Kerberos and DNS + - hostname_prefix: "freeipa" + count: 1 + subnet_index: 0 # TODO: Implement better selection of AZs. this is the combined list of public + private + groups: + - freeipa + - jump_host + - nodes + instance_type: m5.large + # Manager, Master, DB, and CA + - hostname_prefix: "base" # hostname will be of form {{ name_prefix }}-{{ hostname_prefix }}- + count: 1 + groups: + - cloudera_manager + - cluster_masters + - cluster + - nodes + - db_server + - linux_nodes + - deployment + host_template: Masters + instance_type: m5.4xlarge + root_volume: + volume_size: 250 + subnet_index: 1 + tls: True + # Cluster Worker + - hostname_prefix: worker + count: 2 + groups: + - cluster_workers + - cluster + - nodes + - linux_nodes + - deployment + instance_type: c5.2xlarge + subnet_index: 1 + root_volume: + volume_size: 250 + host_template: Workers + tls: True + # ECS Master + - hostname_prefix: ecs-m + count: 1 + groups: + - cluster_ecs_masters + - cluster + - nodes + - ecs_nodes + - small_ecs_nodes + - linux_nodes + - deployment + instance_type: c5.2xlarge + subnet_index: 1 + root_volume: + volume_size: 250 + volumes: + - device: /dev/sdb + mount: /ecs + size: 250 + type: gp2 + host_template: ECSMaster + tls: True + # ECS Workers + - hostname_prefix: ecs-w + count: 3 + groups: + - cluster_ecs_workers + - cluster + - nodes + - ecs_nodes + - linux_nodes + - deployment + instance_type: m5.8xlarge + subnet_index: 2 + root_volume: + volume_size: 250 + volumes: + - device: /dev/sdb + mount: /ecs + size: 250 + type: gp2 + host_template: ECSWorker + tls: True diff --git a/private-cloud/ecs-on-aws/execution-environment.yml b/private-cloud/ecs-on-aws/execution-environment.yml new file mode 100644 index 0000000..dfb823a --- /dev/null +++ b/private-cloud/ecs-on-aws/execution-environment.yml @@ -0,0 +1,18 @@ +--- + +version: 3 + +images: + base_image: + name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections + +dependencies: + galaxy: + collections: + - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git + type: git + version: devel + - name: freeipa.ansible_freeipa + version: 1.11.0 + python: + - dnspython diff --git a/private-cloud/ecs-on-aws/external_setup.yml b/private-cloud/ecs-on-aws/external_setup.yml new file mode 100644 index 0000000..cdbe151 --- /dev/null +++ b/private-cloud/ecs-on-aws/external_setup.yml @@ -0,0 +1,48 @@ +--- + +- name: Set up CDP Private Cloud external prerequisites + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up external prerequisites for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_ext.yml + +# NOTE: Cluster hosts become unavilable after krb5 client setup +# This is a temporary workaround while we debug the source of the issue +- name: Reboot all cluster hosts + hosts: cluster + gather_facts: no + become: yes + tasks: + - name: Reboot all cluster hosts + ansible.builtin.reboot: \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/group_vars/all.yml b/private-cloud/ecs-on-aws/group_vars/all.yml new file mode 100644 index 0000000..19f3184 --- /dev/null +++ b/private-cloud/ecs-on-aws/group_vars/all.yml @@ -0,0 +1,3 @@ +--- +# If needed, set the following variable to tell Ansible which key to use +# ansible_ssh_private_key_file: ~/.ssh/some-private-key diff --git a/private-cloud/ecs-on-aws/hostvars.j2 b/private-cloud/ecs-on-aws/hostvars.j2 new file mode 100644 index 0000000..bf2ca91 --- /dev/null +++ b/private-cloud/ecs-on-aws/hostvars.j2 @@ -0,0 +1,9 @@ +{# Collect and output individual host variables #} +{% macro host_variables(host) %} +{% set fields = [] %} +{% set _ = fields.append("ansible_user=" + host['ansible_user']) if 'ansible_user' in host %} +{% set _ = fields.append("host_template=" + host['host_template']) if 'host_template' in host %} +{% set _ = fields.append("label=" + host['label']) if 'label' in host %} +{% set _ = fields.append("tls=" + host['tls'] | string) if 'tls' in host %} +{{ host['inventory_hostname'] }} {{ fields | join(' ') }} +{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/instance_vars.j2 b/private-cloud/ecs-on-aws/instance_vars.j2 new file mode 100644 index 0000000..6e7063f --- /dev/null +++ b/private-cloud/ecs-on-aws/instance_vars.j2 @@ -0,0 +1,13 @@ +{# Define the metadata tags for the individual Openstack instances #} +{# Output should be TF map _entries_, not a map itself #} + +{% macro instance_tags(host) %} +{% set tags = {} %} +{% set _ = tags.update({ 'ansible_user': host.ansible_user }) if host.ansible_user is defined %} +{% set _ = tags.update({ 'host_template': host.host_template }) if host.host_template is defined %} +{% set _ = tags.update({ 'groups': host.groups | join(', ') }) if host.groups is defined %} +{% set _ = tags.update({ 'tls': host.tls | string }) if host.tls is defined %} +{% for k, v in tags.items() %} + {{ k }} = "{{ v }}"{{ "," if not loop.last else "" }} +{% endfor %} +{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/internal_setup.yml b/private-cloud/ecs-on-aws/internal_setup.yml new file mode 100644 index 0000000..18b56ea --- /dev/null +++ b/private-cloud/ecs-on-aws/internal_setup.yml @@ -0,0 +1,38 @@ +--- + +- name: Set up CDP Private Cloud internal prerequisites + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up internal prerequisites for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_int.yml diff --git a/private-cloud/ecs-on-aws/pre_setup.yml b/private-cloud/ecs-on-aws/pre_setup.yml new file mode 100644 index 0000000..16b31b0 --- /dev/null +++ b/private-cloud/ecs-on-aws/pre_setup.yml @@ -0,0 +1,231 @@ +--- + +# Run variable checker + +# Provision the infrastructure at the cloud provider +- name: Provision infrastructure resources + hosts: localhost + connection: local + gather_facts: yes + module_defaults: + # Set your definition-specific inventory host variables here. + # NOTE: the 'node' variable is in scope and contains each entry coming from + # the Terraform 'nodes' output variable. + ansible.builtin.add_host: + host_template: "{{ node.metadata.host_template | default(omit) }}" + tls: "{{ node.metadata.tls | default(omit) }}" + # TODO Review the SSH args, these are overwritten + ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -C -o ControlMaster=auto -o ControlPersist=1200s -o BatchMode=yes' + tags: infra + tasks: + - name: Provision the infrastructure resources in the cloud provider + when: infra_type != 'static' + ansible.builtin.import_role: + name: cloudera.infra.provision + vars: + provision_provider: "{{ infra_type }}" + provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" + provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" + provision_terraform_parallelism: "{{ terraform.parallelism | default(omit) }}" + provision_state_storage: "{{ terraform.state.storage }}" + provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" + provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" + provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" + provision_name_prefix: "{{ name_prefix }}" + provision_domain_suffix: "{{ domain }}" + provision_ssh_keypair_label: "{{ ssh_keypair.label | default(public_key_id) }}" + provision_ssh_keypair_public_key: "{{ ssh_keypair.public_key | default(public_key_text) }}" + provision_owner_email: "{{ owner_email }}" + provision_openstack_network_name: "{{ infra[infra_type].network_name | default(omit) }}" + provision_openstack_default_image_name: "{{ infra[infra_type].default_image_name | default(omit) }}" + provision_openstack_default_availability_zone: "{{ infra[infra_type].default_availability_zone | default(omit) }}" + provision_aws_ec2_region: "{{ infra[infra_type].region | default(omit) }}" + provision_aws_ec2_vpc_enable_dns_support: "{{ infra[infra_type].enable_dns | default(omit) }}" + provision_aws_ec2_vpc_enable_dns_hostnames: "{{ infra[infra_type].enable_dns | default(omit) }}" + provision_aws_ec2_public_subnets: "{{ infra[infra_type].public_subnets | default(omit) }}" + provision_aws_ec2_private_subnets: "{{ infra[infra_type].private_subnets | default(omit) }}" + provision_aws_ec2_default_security_group_ingress: "{{ infra[infra_type].default_security_group.ingress | default(omit) }}" + provision_instances: "{{ infra[infra_type].nodes | default([]) }}" + provision_tags: "{{ deployment_tags | default(omit) }}" + +# Confirm the availablity of all nodes +- name: Ensure node readiness + hosts: nodes + gather_facts: no + tasks: + - name: Ensure the node is reachable + ansible.builtin.wait_for_connection: + timeout: 60 + +# Prepare and mount any attached volumes +- name: Prepare and mount storage volumes + hosts: nodes + gather_facts: no + become: True + tasks: + - name: Prepare storage volumes + when: infra_type != "static" and storage_volumes | length > 0 + ansible.builtin.import_role: + name: cloudera.infra.mount + vars: + mount_volumes: "{{ storage_volumes }}" + mount_provider: "{{ infra_type }}" + +# Prepare the FreeIPA sidecar services +- name: Provision FreeIPA services + hosts: freeipa + gather_facts: yes + become: yes + vars: + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + # If not set specifically: if AWS, then VPC CIDR + 2 (see https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) + resolv_nameservers: "{{ upstream_nameservers | default(default_dns[infra_type]) }}" + default_dns: + aws: "{{ [ hostvars.localhost.provision.vpc.cidr | ansible.utils.ipmath(2) ] }}" + module_defaults: + freeipa.ansible_freeipa.ipadnszone: + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + freeipa.ansible_freeipa.ipadnsrecord: + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + tasks: + - name: Set up the FreeIPA server + ansible.builtin.import_role: + name: cloudera.exe.freeipa_server + vars: + ipaserver_hostname: "{{ inventory_hostname }}" + ipaserver_realm: "{{ freeipa.realm }}" + ipaserver_domain: "{{ name_prefix }}.{{ domain }}" + ipaserver_no_host_dns: yes + ipaserver_setup_firewalld: no + ipaserver_setup_dns: "{{ enable_dns }}" + ipaserver_resolv_nameservers: "{{ resolv_nameservers }}" + ipaserver_auto_reverse: "{{ enable_dns | ternary(True, omit) }}" + ipaserver_no_forwarders: "{{ enable_dns | ternary(True, omit) }}" + ipaserver_forward_policy: "{{ enable_dns | ternary('only', omit) }}" + ipaserver_recursion_acl_cidr: "{{ enable_dns | ternary(hostvars.localhost.provision.vpc.cidr, omit) }}" + ipaserver_copy_csr_to_controller: yes + ipaclient_mkhomedir: yes + ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" # TODO Add test for these parameters - error in role is opaque + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + + - name: Configure provisioned FreeIPA global DNS + when: enable_dns + block: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in provisioned FreeIPA service + freeipa.ansible_freeipa.ipadnszone: + zone_name: "{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + forward_policy: none + + - name: Create reverse DNS zone for '{{ hostvars.localhost.provision.vpc.cidr }}' + freeipa.ansible_freeipa.ipadnszone: + name_from_ip: "{{ hostvars.localhost.provision.vpc.cidr }}" + dynamic_update: yes + allow_sync_ptr: yes + + - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' + when: "'cluster_ecs_masters' in groups" + block: + - name: Ensure DNS zone for ECS 'apps' + freeipa.ansible_freeipa.ipadnszone: + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + forward_policy: none + + - name: Ensure DNS wildcard records for ECS 'apps' and domain + freeipa.ansible_freeipa.ipadnsrecord: + name: "*" + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + record_type: "A" + record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" + create_reverse: no + + - name: Resolve provisioned FreeIPA DNS server addresses + ansible.builtin.set_fact: + dns_server_ips: "{{ dns_server_ips | default([]) | union([ ansible_default_ipv4['address'] ]) }}" + delegate_to: localhost + delegate_facts: true + +# Update existing FreeIPA DNS services +- name: Configure existing FreeIPA global DNS + hosts: localhost + connection: local + gather_facts: no + vars: + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + dns_server: "{{ freeipa.server | default(lookup('ansible.builtin.env', 'IPA_HOST', default='Undefined')) }}" + module_defaults: + community.general.ipa_dnszone: + ipa_host: "{{ dns_server }}" + ipa_user: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default='Undefined')) }}" + ipa_pass: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" + tasks: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in existing FreeIPA service + when: enable_dns and dns_server is defined and 'freeipa' not in groups + block: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' + community.general.ipa_dnszone: + validate_certs: no + dynamicupdate: yes + allowsyncptr: yes + zone_name: "{{ name_prefix }}.{{ domain }}" + state: present + + - name: Create reverse DNS zone for '{{ infra_cidr }}' + community.general.ipa_dnszone: + validate_certs: no + dynamicupdate: yes + allowsyncptr: yes + zone_name: "{{ infra_cidr | ansible.utils.ipaddr('revdns') }}" + state: present + + - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' + when: "'cluster_ecs_masters' in groups" + block: + - name: Ensure DNS zone for ECS 'apps' + community.general.ipa_dnszone: + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + validate_certs: no + + - name: Ensure DNS wildcard records for ECS 'apps' and domain + community.general.ipa_dnszone: + name: "*" + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + record_type: "A" + record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" + validate_certs: no + + - name: Resolve existing FreeIPA DNS server addresses + ansible.builtin.set_fact: + dns_hosts: "{{ [ dns_server ] }}" + dns_server_ips: "{{ [ lookup('community.general.dig', dns_server ) ] }}" + +# Enroll deployment hosts with FreeIPA Kerberos and possibly with DNS +- name: Register hosts with FreeIPA + hosts: nodes:!freeipa + gather_facts: yes + become: True + tasks: + - name: Register host with FreeIPA services + when: krb5_kdc_type == "Red Hat IPA" + ansible.builtin.import_role: + name: cloudera.exe.freeipa_client + vars: + ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" + ipaserver_realm: "{{ freeipa.realm }}" + ipa_hosts: "{{ groups['freeipa'] | default(hostvars.localhost.dns_hosts) }}" + ipa_server_ips: "{{ hostvars.localhost.dns_server_ips }}" + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" + ipaadmin_principal: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default=omit)) }}" + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + +# Prepare SME Lab resources +- name: Prepare TLS and supporting services resources + import_playbook: pre_setup_resources.yml + tags: + - prereq + - infra \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/pre_setup_resources.yml b/private-cloud/ecs-on-aws/pre_setup_resources.yml new file mode 100644 index 0000000..21d0321 --- /dev/null +++ b/private-cloud/ecs-on-aws/pre_setup_resources.yml @@ -0,0 +1,16 @@ +--- + # Playbook to setup additional resources for PvC Deployment + +- name: Ensure additional requirements are available + hosts: nodes + gather_facts: no + become: yes + tasks: + + - name: Install required packages + ansible.builtin.package: + name: + - tree + - jq + - zip + state: present diff --git a/private-cloud/ecs-on-aws/pre_teardown.yml b/private-cloud/ecs-on-aws/pre_teardown.yml new file mode 100644 index 0000000..68c014f --- /dev/null +++ b/private-cloud/ecs-on-aws/pre_teardown.yml @@ -0,0 +1,24 @@ +--- + +- name: Deprovision the infrastructure resources + hosts: localhost + connection: local + gather_facts: no + tags: + - teardown + - provision + tasks: + - name: Deprovision the infrastructure resources in the cloud provider + when: groups['all'] | length > 0 + ansible.builtin.import_role: + name: cloudera.infra.provision + vars: + provision_state: absent + provision_provider: "{{ infra_type }}" + provision_inventory_file: inventory_static.ini + provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" + provision_state_storage: "{{ terraform.state.storage }}" + provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" + provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" + provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" + provision_name_prefix: "{{ name_prefix }}" diff --git a/private-cloud/ecs-on-aws/validate_dns_lookups.yml b/private-cloud/ecs-on-aws/validate_dns_lookups.yml new file mode 100644 index 0000000..236169e --- /dev/null +++ b/private-cloud/ecs-on-aws/validate_dns_lookups.yml @@ -0,0 +1,27 @@ +- name: Check that all hosts in inventory are reachable + hosts: all + gather_facts: no + tasks: + - name: Ping each host from controller + ping: + + +- name: Check Forward and Reverse DNS lookups + hosts: localhost + connection: local + gather_facts: no + tasks: + + - name: Print the list of hosts + debug: + var: groups['all'] + + - name: Forward lookup + ansible.builtin.debug: + msg: + - "Forward lookup DNS for {{ item }} is {{ lookup('community.general.dig', item , qtype='A') }}" + loop: "{{ groups['all'] }}" + + # TODO Register forward lookup IP and do reverse lookup + + From 64b921afddda62225c17ef9f32b32d3fb233377a Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:30:59 -0500 Subject: [PATCH 02/45] Update README and revert to cloudera.infra FreeIPA roles Signed-off-by: Jim Enright --- private-cloud/ecs-on-aws/README.md | 16 ++++++++-------- private-cloud/ecs-on-aws/ansible-navigator.yml | 1 - private-cloud/ecs-on-aws/pre_setup.yml | 6 ++++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md index d5d6d04..b6bcd7a 100644 --- a/private-cloud/ecs-on-aws/README.md +++ b/private-cloud/ecs-on-aws/README.md @@ -58,8 +58,8 @@ Configuration is passed via environment variables and an user-facing configurati | Variable | Description | Status | |----------|-------------|--------| - | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/wmudge/.ssh/demo_ops.pub` | Mandatory | - | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/wmudge/Documents/webster_mudge_2022_2023_cloudera_license.txt` | Mandatory | + | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/example/.ssh/demo_ops.pub` | Mandatory | + | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/example/Documents/example_cloudera_license.txt` | Mandatory | | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. Used also for remote storage of Terraform state in AWS. | Mandatory | @@ -127,21 +127,21 @@ Run the following: ```bash # Run the 'external' system configuration ansible-navigator run external_setup.yml \ - -e @lab_configs/config.yml \ + -e @config.yml \ -i inventory_static__aws.ini ``` ```bash # Run the 'internal' Cloudera installations and configurations ansible-navigator run internal_setup.yml \ - -e @lab_configs/config.yml \ + -e @config.yml \ -i inventory_static__aws.ini ``` ```bash # Run the Cloudera cluster configuration and imports ansible-navigator run base_setup.yml \ - -e @lab_configs/config.yml \ + -e @config.yml \ -i inventory_static__aws.ini ``` @@ -149,7 +149,7 @@ And lastly, the _postfix_: ```bash ansible-navigator run base_postfix.yml \ - -e @lab_configs/config.yml \ + -e @config.yml \ -i inventory_static__aws.ini ``` @@ -175,7 +175,7 @@ Run the following: ```bash ansible-navigator run pre_teardown.yml \ - -e @lab_configs/config.yml \ + -e @config.yml \ -e @definition.yml \ -i inventory_static__aws.ini ``` @@ -186,7 +186,7 @@ You can also run `terraform destroy` within the `tf_deployment_*` directory. ### Confirm SSH Connectivity -Run the following from _the root of the definition project, i.e. `demo-definitions`_: +Run the following: ```bash ansible -m ansible.builtin.ping -i inventory_static__aws.ini all diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml index d60d305..e1c62c0 100644 --- a/private-cloud/ecs-on-aws/ansible-navigator.yml +++ b/private-cloud/ecs-on-aws/ansible-navigator.yml @@ -42,7 +42,6 @@ ansible-navigator: - src: "~/.aws" dest: "/runner/.aws" options: "Z" - options: "Z" - src: "~/.ssh" dest: "/runner/.ssh" options: "Z" diff --git a/private-cloud/ecs-on-aws/pre_setup.yml b/private-cloud/ecs-on-aws/pre_setup.yml index 16b31b0..fcf0bea 100644 --- a/private-cloud/ecs-on-aws/pre_setup.yml +++ b/private-cloud/ecs-on-aws/pre_setup.yml @@ -90,7 +90,8 @@ tasks: - name: Set up the FreeIPA server ansible.builtin.import_role: - name: cloudera.exe.freeipa_server + # name: cloudera.exe.freeipa_server + name: cloudera.infra.freeipa_server vars: ipaserver_hostname: "{{ inventory_hostname }}" ipaserver_realm: "{{ freeipa.realm }}" @@ -213,7 +214,8 @@ - name: Register host with FreeIPA services when: krb5_kdc_type == "Red Hat IPA" ansible.builtin.import_role: - name: cloudera.exe.freeipa_client + # name: cloudera.exe.freeipa_client + name: cloudera.infra.freeipa_client vars: ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" ipaserver_realm: "{{ freeipa.realm }}" From 0a08531914176c10b9cbfd6748a6a5825d6dc146 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:30:59 -0500 Subject: [PATCH 03/45] Update database version and README Signed-off-by: Jim Enright --- private-cloud/ecs-on-aws/README.md | 5 ++++- private-cloud/ecs-on-aws/definition.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md index b6bcd7a..df7ec25 100644 --- a/private-cloud/ecs-on-aws/README.md +++ b/private-cloud/ecs-on-aws/README.md @@ -161,7 +161,10 @@ Once the cluster is up, you can access all of the UIs within, including the Free ssh -D -q -C -N @ ``` -and use a SOCKS5 proxy switcher in your browser. You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. +and use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega browser extension). +In the SOCKS5 proxy configuration, set Protocol to SOCKS5; Server to localhost and Port to 8157. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. + +You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! diff --git a/private-cloud/ecs-on-aws/definition.yml b/private-cloud/ecs-on-aws/definition.yml index 17d9a04..a9c6d6a 100644 --- a/private-cloud/ecs-on-aws/definition.yml +++ b/private-cloud/ecs-on-aws/definition.yml @@ -79,7 +79,7 @@ license_local_tmp_path: /tmp/cloudera_license.txt ### RDBMS ### ############### database_type: postgresql -database_version: 10 +database_version: 12 database_tls: true #########++++ From 4ed2f1cc3a64ae56db1e0477b809a7cdd70af81b Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:30:59 -0500 Subject: [PATCH 04/45] Add example for PvC Base on AWS Signed-off-by: Jim Enright --- private-cloud/pvc-base-on-aws/.gitignore | 17 ++ private-cloud/pvc-base-on-aws/README.md | 198 +++++++++++++++ .../pvc-base-on-aws/ansible-navigator.yml | 49 ++++ .../pvc-base-on-aws/base_postfix.yml | 38 +++ private-cloud/pvc-base-on-aws/base_setup.yml | 38 +++ private-cloud/pvc-base-on-aws/cluster.yml | 228 +++++++++++++++++ private-cloud/pvc-base-on-aws/config.yml | 18 ++ private-cloud/pvc-base-on-aws/definition.yml | 198 +++++++++++++++ .../pvc-base-on-aws/execution-environment.yml | 18 ++ .../pvc-base-on-aws/external_setup.yml | 48 ++++ .../pvc-base-on-aws/group_vars/all.yml | 3 + private-cloud/pvc-base-on-aws/hostvars.j2 | 9 + .../pvc-base-on-aws/instance_vars.j2 | 13 + .../pvc-base-on-aws/internal_setup.yml | 38 +++ private-cloud/pvc-base-on-aws/pre_setup.yml | 233 ++++++++++++++++++ .../pvc-base-on-aws/pre_setup_resources.yml | 16 ++ .../pvc-base-on-aws/pre_teardown.yml | 24 ++ .../pvc-base-on-aws/validate_dns_lookups.yml | 27 ++ 18 files changed, 1213 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/.gitignore create mode 100644 private-cloud/pvc-base-on-aws/README.md create mode 100644 private-cloud/pvc-base-on-aws/ansible-navigator.yml create mode 100644 private-cloud/pvc-base-on-aws/base_postfix.yml create mode 100644 private-cloud/pvc-base-on-aws/base_setup.yml create mode 100644 private-cloud/pvc-base-on-aws/cluster.yml create mode 100644 private-cloud/pvc-base-on-aws/config.yml create mode 100644 private-cloud/pvc-base-on-aws/definition.yml create mode 100644 private-cloud/pvc-base-on-aws/execution-environment.yml create mode 100644 private-cloud/pvc-base-on-aws/external_setup.yml create mode 100644 private-cloud/pvc-base-on-aws/group_vars/all.yml create mode 100644 private-cloud/pvc-base-on-aws/hostvars.j2 create mode 100644 private-cloud/pvc-base-on-aws/instance_vars.j2 create mode 100644 private-cloud/pvc-base-on-aws/internal_setup.yml create mode 100644 private-cloud/pvc-base-on-aws/pre_setup.yml create mode 100644 private-cloud/pvc-base-on-aws/pre_setup_resources.yml create mode 100644 private-cloud/pvc-base-on-aws/pre_teardown.yml create mode 100644 private-cloud/pvc-base-on-aws/validate_dns_lookups.yml diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/pvc-base-on-aws/.gitignore new file mode 100644 index 0000000..e5a1363 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/.gitignore @@ -0,0 +1,17 @@ +# Ansible Navigator assets +ansible-navigator.log +runs +context + +# Terraform deployments +tf_deployment* + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Static inventory files +inventory_static* \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/pvc-base-on-aws/README.md new file mode 100644 index 0000000..1e4bebf --- /dev/null +++ b/private-cloud/pvc-base-on-aws/README.md @@ -0,0 +1,198 @@ +# PvC Base Cluster on AWS + +> Constructs a CDP Private Cloud Base cluster running on AWS. + +## Requirements + +This example requires an execution environment with dependencies to run the automation; and a set of configuration variables. + +We provide instructions for using a container based execution environment. + +### Container Execution Environment + +1. Create and activate a new `virtualenv` and install `ansible-core` and `ansible-navigator` + + ```bash + python -m venv ~/cdp-navigator; + + source ~/cdp-navigator/bin/activate; + + pip install ansible-core==2.12.10 ansible-navigator==3.4.0 + ``` + +1. Clone this repository. + + ```bash + git clone https://github.com/cloudera-labs/cloudera-deploy.git; + ``` + +1. Change your working directory to this project. + + ```bash + cd cloudera-deploy/private-cloud/pvc-base-on-aws + ``` + +1. We currently need to build a local `cldr-runner` image for use as an Execution Environment. _(This is necessary at this point in the release cycle. It will become optional.)_ + + ```bash + ansible-navigator builder build --prune-images -v 3 --tag ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + ``` + +This step sometimes takes a ~~number of minutes~~ long time to complete. YMMV. + +Once constructed, you can check the image by running `ansible-navigator` and using the prompts to examine: + +```bash +ansible-navigator images +``` + +Or by running the Docker command directly, `docker image ls`. + +### Configuration Variables + +Configuration is passed via environment variables and an user-facing configuration file. + +#### Environment Variables + +* Set up the following definition environment variables: + + | Variable | Description | Status | + |----------|-------------|--------| + | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/example/.ssh/demo_ops.pub` | Mandatory | + | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/example/Documents/example_cloudera_license.txt` | Mandatory | + | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | + | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | + | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. Used also for remote storage of Terraform state in AWS. | Mandatory | + +> **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. + + +#### Configuration file variables + +Edit the `config.yml` user-facing configuration file to match your particular deployment. + +*NOTE:* `name_prefix` should be 4-8 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. + +```yaml +name_prefix: "labaw" # CHANGE THIS +owner_prefix: "pvc-base" +owner_email: "example@cloudera.com" +infra_region: "eu-west-1" +infra_type: "aws" # "aws", "static" +domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) +realm: "CLDR.EXAMPLE" # The Kerberos realm +common_password: "Example776" +admin_password: "Example776" +deployment_tags: + owner: "{{ owner_prefix }}" + email: "{{ owner_email }}" + project: "PvC Base Lab - {{ owner_prefix }}-{{ name_prefix }}" + enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy +``` + +## Execution + +### Pre-setup Playbook + +This definition-specific playbook includes tasks such as: +* Instructure provisioning +* FreeIPA DNS and KRB services provisioning + +Run the following command + +```bash +ansible-navigator run pre_setup.yml \ +-e @config.yml \ +-e @definition.yml +``` + +Once the pre-setup playbook completes confirm that: + +* You can connect to each node via the inventory - see Confirm SSH Connectivity. Note that a A `validate_dns_lookups.yml` Playbook exists to check connectivity. +* Connect to FreeIPA UI and login with the `IPA_USER` and `IPA_PASSWORD` credentials in the configuration file. See Cluster Access for more details. + +### Platform Playbooks + +These playbooks configure and deploy PVC Base. They use the infrastructure provisioned (or assigned, if using `static` inventory). + +Tasks include: +* System/host configuration +* Cloudera Manager server and agent installation and configuration +* Cluster template imports + +Run the following: + +```bash +# Run the 'external' system configuration +ansible-navigator run external_setup.yml \ + -e @config.yml \ + -i inventory_static__aws.ini +``` + +```bash +# Run the 'internal' Cloudera installations and configurations +ansible-navigator run internal_setup.yml \ + -e @config.yml \ + -i inventory_static__aws.ini +``` + +```bash +# Run the Cloudera cluster configuration and imports +ansible-navigator run base_setup.yml \ + -e @config.yml \ + -i inventory_static__aws.ini +``` + +And lastly, the _postfix_: + +```bash +ansible-navigator run base_postfix.yml \ + -e @config.yml \ + -i inventory_static__aws.ini +``` + +## Cluster Access + +Once the cluster is up, you can access all of the UIs within, including the FreeIPA sidecar, via a SSH tunnel: + +```bash +ssh -D -q -C -N @ +``` + +and use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega browser extension). +In the SOCKS5 proxy configuration, set Protocol to SOCKS5; Server to localhost and Port to 8157. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. + +You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. + +In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! + +```bash +ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @ +``` + +## Teardown + +Run the following: + +```bash +ansible-navigator run pre_teardown.yml \ + -e @config.yml \ + -e @definition.yml \ + -i inventory_static__aws.ini +``` + +You can also run `terraform destroy` within the `tf_deployment_*` directory. + +## Troubleshooting + +### Confirm SSH Connectivity + +Run the following: + +```bash +ansible -m ansible.builtin.ping -i inventory_static__aws.ini all +``` + +This will check to see if the inventory file is well constructed, etc. diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml new file mode 100644 index 0000000..e1c62c0 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -0,0 +1,49 @@ +--- + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - AWS_PROFILE + - CDP_PROFILE + - SSH_PUBLIC_KEY_FILE + - CDP_LICENSE_FILE + - IPA_USER + - IPA_PASSWORD + set: + ANSIBLE_SSH_CONTROL_PATH: "/dev/shm/cp%%h-%%p-%%r" + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + pull: + arguments: + - "--tls-verify=false" + volume-mounts: + - src: "${SSH_PUBLIC_KEY_FILE}" + dest: "${SSH_PUBLIC_KEY_FILE}" + - src: "${CDP_LICENSE_FILE}" + dest: "${CDP_LICENSE_FILE}" + - src: "~/.aws" + dest: "/runner/.aws" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/private-cloud/pvc-base-on-aws/base_postfix.yml b/private-cloud/pvc-base-on-aws/base_postfix.yml new file mode 100644 index 0000000..c7336f5 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/base_postfix.yml @@ -0,0 +1,38 @@ +--- + +- name: Postfix CDP Private Cloud clusters + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Postfix clusters for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_postfix.yml diff --git a/private-cloud/pvc-base-on-aws/base_setup.yml b/private-cloud/pvc-base-on-aws/base_setup.yml new file mode 100644 index 0000000..be6790b --- /dev/null +++ b/private-cloud/pvc-base-on-aws/base_setup.yml @@ -0,0 +1,38 @@ +--- + +- name: Set up CDP Private Cloud clusters + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up clusters for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_setup.yml diff --git a/private-cloud/pvc-base-on-aws/cluster.yml b/private-cloud/pvc-base-on-aws/cluster.yml new file mode 100644 index 0000000..a59f68b --- /dev/null +++ b/private-cloud/pvc-base-on-aws/cluster.yml @@ -0,0 +1,228 @@ +--- + +clusters: + - name: PVC-Base + type: base + services: [ATLAS, HBASE, HDFS, HIVE, HIVE_ON_TEZ, INFRA_SOLR, KAFKA, OZONE, RANGER, SPARK_ON_YARN, TEZ, YARN, ZOOKEEPER] + security: + kerberos: true + tls: true + repositories: "{{ repositories }}" + configs: + ATLAS: + SERVICEWIDE: + kerberos.auth.enable: true + ATLAS_SERVER: + atlas_authentication_method_file: true + atlas_admin_password: "{{ common_password }}" + atlas_sso_knox_enabled: false + atlas_authentication_method_trustedproxy: false + atlas_authentication_method_pam: false + atlas_authentication_method_ldap: true + atlas_authentication_method_ldap_url: "{{ auth_provider.ldap_url }}" + atlas_authentication_method_ldap_bind_dn: "{{ auth_provider.ldap_bind_user_dn }}" + atlas_authentication_method_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" + atlas_authentication_method_ldap_userDNpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" + atlas_authentication_method_ldap_base_dn: "{{ auth_provider.ldap_search_base.user }}" + atlas_authentication_method_ldap_user_searchfilter: "{{ auth_provider.ldap_search_filter.user }}" + atlas_authentication_method_ldap_groupSearchBase: "{{ auth_provider.ldap_search_base.group }}" + atlas_authentication_method_ldap_groupSearchFilter: "{{ auth_provider.ldap_search_filter.member }}" + atlas_authentication_method_ldap_groupRoleAttribute: "{{ auth_provider.ldap_attr_group_name }}" + atlas_authentication_method_ldap_type: "ldap" + atlas_authentication_method_ldap_default_role: "ROLE_ADMIN" + HBASE: + SERVICEWIDE: + hadoop_secure_web_ui: true + hbase_graceful_stop_timeout: 5 + REGIONSERVER: + hbase_regionserver_java_heapsize: 268435456 + MASTER: + hbase_master_java_heapsize: 134217728 + GATEWAY: + hbase_client_java_heapsize: 134217728 + # NOTE: Have removed /mnt from the various HFDS directories + HDFS: + SERVICEWIDE: + hdfs_verify_ec_with_topology_enabled: false + enable_ranger_authorization: true + hadoop_secure_web_ui: false + trusted_realms: "{{ realm | upper }}, {{ realm | lower }}" + DATANODE: + datanode_java_heapsize: 134217728 + dfs_data_dir_list: /dfs/dn + dfs_datanode_max_locked_memory: 0 + dfs_datanode_failed_volumes_tolerated: 0 + GATEWAY: + hdfs_client_java_heapsize: 134217728 + NAMENODE: + dfs_name_dir_list: /dfs/nn + namenode_java_heapsize: 134217728 + role_config_suppression_namenode_java_heapsize_minimum_validator: true + SECONDARYNAMENODE: + secondary_namenode_java_heapsize: 134217728 + fs_checkpoint_dir_list: /dfs/snn + HIVE: + SERVICEWIDE: + ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" + GATEWAY: + hive_client_java_heapsize: 134217728 + HIVEMETASTORE: + hive_metastore_java_heapsize: 268435456 + metastore_canary_health_enabled: false + HIVE_ON_TEZ: + SERVICEWIDE: + ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" + KAFKA: + SERVICEWIDE: + kerberos.auth.enable: true + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + producer.metrics.enable: false + service_config_suppression_kafka_broker_count_validator: true + service_config_suppression_offsets.topic.replication.factor: true + service_config_suppression_transaction.state.log.replication.factor: true + KAFKA_BROKER: + graceful_stop_timeout: 5 + OZONE: + SERVICEWIDE: + ozone.security.enabled: true + ozone.service.id: ozone1 + ozone.security.http.kerberos.enabled: false + service_config_suppression_ozone_manager_count_validator: true + service_config_suppression_storage_container_manager_count_validator: true + OZONE_MANAGER: + om_max_heap_size: 2048 + STORAGE_CONTAINER_MANAGER: + scm_max_heap_size: 2048 + OZONE_DATANODE: + ozone_datanode_heap_size: 2048 + OZONE_PROMETHEUS: + ozone.prometheus.http-port: 19090 + RANGER: + SERVICEWIDE: + keyadmin_user_password: "{{ common_password }}" + rangeradmin_user_password: "{{ common_password }}" + rangertagsync_user_password: "{{ common_password }}" + rangerusersync_user_password: "{{ common_password }}" + RANGER_ADMIN: + ranger_admin_max_heap_size: 1024 + ranger_authentication_method: "LDAP" + ranger.ldap.url: "{{ auth_provider.ldap_url }}" + ranger.ldap.bind.dn: "{{ auth_provider.ldap_bind_user_dn }}" + ranger_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" + ranger.ldap.user.dnpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" + ranger.ldap.base.dn: "{{ auth_provider.ldap_base_dn }}" + ranger.ldap.user.searchfilter: "(&(sAMAccountName={0})(objectClass=person))" + ranger.ldap.group.searchfilter: "(&(member={0})(objectClass=group))" + ranger.ldap.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" + ranger.ldap.group.roleattribute: "{{ auth_provider.ldap_attr_group_name }}" + ranger.ldap.referral: "follow" + RANGER_TAGSYNC: + ranger_tagsync_max_heap_size: 1024 + RANGER_USERSYNC: + ranger_usersync_max_heap_size: 1024 + ranger.usersync.ldap.binddn: "{{ auth_provider.ldap_bind_user_dn }}" + ranger.usersync.ldap.url: "{{ auth_provider.ldap_url }}" + ranger_usersync_ldap_ldapbindpassword: "{{ auth_provider.ldap_bind_password }}" + ranger.usersync.ldap.referral: follow + ranger.usersync.ldap.username.caseconversion: lower + ranger.usersync.ldap.groupname.caseconversion: lower + ranger.usersync.ldap.grouphierarchylevels: 0 + ranger.usersync.group.based.role.assignment.rules: '&ROLE_SYS_ADMIN:g:field-se' + ranger.usersync.group.memberattributename: "{{ auth_provider.ldap_attr_group_membership }}" + ranger.usersync.group.nameattribute: "{{ auth_provider.ldap_attr_group_name }}" + ranger.usersync.group.objectclass: group + ranger.usersync.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" +# ranger.usersync.group.searchfilter: "" +# ranger.usersync.ldap.user.searchfilter: "" + ranger.usersync.ldap.user.nameattribute: "{{ auth_provider.ldap_attr_user_name }}" + ranger.usersync.ldap.user.objectclass: person + ranger.usersync.ldap.user.searchbase: "{{ auth_provider.ldap_search_base.user }}" + ranger.usersync.source.impl.class: org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder + ranger.usersync.user.searchenabled: 'true' + +#obsolete in 7.1.6 ranger.usersync.ldap.searchBase: "{{ auth_provider.ldap_base_dn }}" +#obsolete in 7.1.6 ranger.usersync.ldap.user.groupnameattribute: "{{ auth_provider.ldap_attr_group_name }}" +#obsolete in 7.1.6 ranger.usersync.group.search.first.enabled: 'true' +#obsolete in 7.1.6 ranger.usersync.group.searchenabled: 'true' + INFRA_SOLR: + SERVICEWIDE: + #set false for 7.1.7 since it doesnt build cm_solr + enable_ranger_authorization: false + SOLR_SERVER: + solr_graceful_stop_timeout: 5 + SPARK_ON_YARN: + SPARK_YARN_HISTORY_SERVER: + ssl_enabled: true + history_server_spnego_enabled: false + YARN: + SERVICEWIDE: + hadoop_secure_web_ui: false + JOBHISTORY: + mr2_jobhistory_java_heapsize: 134217728 + NODEMANAGER: + node_manager_java_heapsize: 134217728 + yarn_nodemanager_resource_memory_mb: 2048 + yarn_nodemanager_resource_cpu_vcores: 2 + RESOURCEMANAGER: + resource_manager_java_heapsize: 134217728 + yarn_scheduler_maximum_allocation_mb: 2048 + yarn_scheduler_maximum_allocation_vcores: 2 + ZOOKEEPER: + SERVICEWIDE: + zookeeper_datadir_autocreate: true + service_config_suppression_server_count_validator: true + SERVER: + zookeeper_server_java_heapsize: 134217728 + host_templates: + Masters: + ATLAS: [ATLAS_SERVER] + HBASE: [MASTER, REGIONSERVER, GATEWAY] + HDFS: [NAMENODE, SECONDARYNAMENODE, DATANODE] + HIVE: [HIVEMETASTORE, GATEWAY] + HIVE_ON_TEZ: [HIVESERVER2, GATEWAY] + INFRA_SOLR: [SOLR_SERVER] + KAFKA: [KAFKA_BROKER] + OZONE: [GATEWAY, OZONE_MANAGER, OZONE_RECON, STORAGE_CONTAINER_MANAGER, OZONE_DATANODE, S3_GATEWAY] + RANGER: [RANGER_ADMIN, RANGER_TAGSYNC, RANGER_USERSYNC] + SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER, GATEWAY] + TEZ: [GATEWAY] + YARN: [RESOURCEMANAGER, NODEMANAGER, JOBHISTORY, GATEWAY] + ZOOKEEPER: [SERVER] + + Workers: + HBASE: [REGIONSERVER, GATEWAY] + HDFS: [DATANODE] + HIVE: [GATEWAY] + HIVE_ON_TEZ: [GATEWAY] + OZONE: [OZONE_DATANODE, GATEWAY] + TEZ: [GATEWAY] + SPARK_ON_YARN: [GATEWAY] + YARN: [NODEMANAGER, GATEWAY] + ZOOKEEPER: [SERVER] + +mgmt: + name: Cloudera Management Service + services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] + security: + tls: true + configs: + HOSTMONITOR: + firehose_heapsize: 1073741824 + firehose_non_java_memory_bytes: 2147483648 + SERVICEMONITOR: + firehose_heapsize: 2147483648 + firehose_non_java_memory_bytes: 6442450944 + role_config_suppression_firehose_service_monitor_non_java_memory_role_validator: true + +hosts: + configs: + host_default_proc_memswap_thresholds: + warning: never + critical: never + host_memswap_thresholds: + warning: never + critical: never + host_config_suppression_agent_system_user_group_validator: true + host_config_suppression_memory_overcommitted_validator: true diff --git a/private-cloud/pvc-base-on-aws/config.yml b/private-cloud/pvc-base-on-aws/config.yml new file mode 100644 index 0000000..b36be81 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/config.yml @@ -0,0 +1,18 @@ +--- + +name_prefix: "labaw" # CHANGE THIS +owner_prefix: "pvc-base" +owner_email: "example@cloudera.com" +infra_region: "eu-west-1" +infra_type: "aws" # "aws", "static" +domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) +realm: "CLDR.EXAMPLE" # The Kerberos realm +common_password: "Example776" +admin_password: "Example776" # "Main" default password +deployment_tags: + owner: "{{ owner_prefix }}" + email: "{{ owner_email }}" + project: "PvC Base Lab - {{ owner_prefix }}-{{ name_prefix }}" + enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/pvc-base-on-aws/definition.yml new file mode 100644 index 0000000..478b1d1 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/definition.yml @@ -0,0 +1,198 @@ +--- + +kerberos_activated: True +encryption_activated: True +# tls_activated: True +# autotls: False + +repositories: + # 7.1.8 CHF4 + - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/ +cloudera_manager_version: 7.10.1 + +jdk_version: 11 + +# SSH +public_key_id: "{{ owner_prefix }}-{{ name_prefix }}" +public_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" +public_key_text: "{{ lookup('ansible.builtin.file', lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE')) }}" + +# Cloudera license +license_file: "{{ lookup('ansible.builtin.env', 'CDP_LICENSE_FILE') }}" + +# Terraform settings +terraform: + state: + storage: local + create_remote_storage: False + s3_region: "{{ infra_region }}" + s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" + +# DNS +dns_provider: "freeipa" + +# Connect FreeIPA to Knox and Ranger +freeipa_activated: yes + +# FreeIPA realm settings +freeipa: + realm: "{{ realm }}" + +##### + +# Vars from legacy cloudera-Deploy +use_auto_repo_mirror: no +use_default_cluster_definition: no +use_download_mirror: no +preload_cm_parcel_repo: yes +teardown_everything: yes + +# Defer user and group creation to SSSD/FreeIPA registration +#Use when you dont need ansible to run "ca_server" +skip_user_group_init: no + +# Cloudera Manager details + +cloudera_manager_options: + CUSTOM_BANNER_HTML: "1.5.1 - PvC Base Lab ({{ name_prefix }})" + SESSION_TIMEOUT: 43200 + PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 + # NOTE: Disabled in ChuckL's latest definition + # PHONE_HOME: false + # PROXYUSER_KNOX_GROUPS: "*" + # PROXYUSER_KNOX_HOSTS: "*" + # PROXYUSER_KNOX_PRINCIPAL: "knox" + # PROXYUSER_KNOX_USERS: "*" + KRB_AUTH_ENABLE: "true" + +# License options (this is due to a hardcoded tmp directory on the target/manager node) +license_local_tmp_path: /tmp/cloudera_license.txt + +############### +### RDBMS ### +############### +database_type: postgresql +database_version: 12 +database_tls: true + +#########++++ + +## Red Hat FreeIPA +krb5_kdc_type: Red Hat IPA +krb5_realm: "{{ realm }}" +krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user | default(lookup('env', 'IPA_USER', default='Undefined')) }}@{{ krb5_realm }}" +krb5_kdc_admin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" +krb5_enc_types: "aes256-cts aes128-cts" +# TODO: This needs to either be explicit or rendered from a host with DNS resolution +# Cannot rely on localhost to have DNS access, e.g. private networking on EC2 +#krb5_kdc_host: "{{ lookup('community.general.dig', '_ldap._tcp.' + domain +'./SRV', 'flat=0', wantlist=True, fail_on_error=True) | map(attribute='target') | first }}" +krb5_kdc_host: "{{ groups['freeipa'] | first | default('') }}" + +############################################ +### CM External Auth - FreeIPA as LDAP ### +############################################ + +# Settings for FreeIPA sidecar deployment +cloudera_manager_external_auth: + provider: freeipa + external_first: no + external_only: no + +auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" + +base_dn: "dc={{ (krb5_realm | lower).split('.') | join(',dc=') }}" +user_dn: "cn=users,cn=accounts,{{ base_dn }}" +group_dn: "cn=groups,cn=accounts,{{ base_dn }}" + +auth_providers: + freeipa: + ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}:636" + ldap_base_dn: "{{ base_dn }}" + ldap_bind_user_dn: "uid=admin,{{ user_dn }}" + ldap_bind_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + ldap_search_base: + user: "{{ user_dn }}" + group: "{{ group_dn }}" + ldap_object_class: + user: "person" + group: "groupofnames" + ldap_search_filter: + user: uid={0} + member: (member={0}) + ldap_attr_user_name: "uid" + ldap_attr_group_name: "cn" + ldap_attr_group_membership: "member" + type: LDAP + +## TLS signing +skip_ipa_signing: no # This is the default +remote_ipa_server: "{{ groups['freeipa'] | first | default('') }}" + +tls_key_password: "{{ common_password }}" +tls_keystore_password: "{{ common_password }}" +tls_truststore_password: "{{ common_password }}" + +ca_server_attrs_root: + CN: 'SE-Root CA' +ca_server_attrs_intermediate: + CN: 'SE-Intermediate CA' + +# Infrastructure +infra: + aws: + region: "{{ infra_region }}" + enable_dns: no + public_subnets: # TODO: Implement better selection of AZs + - az: "{{ infra_region }}a" + private_subnets: + - az: "{{ infra_region }}b" + - az: "{{ infra_region }}c" + default_security_group: + ingress: + - cidr: [ "0.0.0.0/0" ] + from: 22 + to: 22 + protocol: TCP + desc: SSH (Ansible) + nodes: + # Sidecar FreeIPA for Kerberos and DNS + - hostname_prefix: "freeipa" + count: 1 + subnet_index: 0 # TODO: Implement better selection of AZs. this is the combined list of public + private + groups: + - freeipa + - jump_host + - nodes + instance_type: m5.large + # Manager, Master, DB, and CA + - hostname_prefix: "base" # hostname will be of form {{ name_prefix }}-{{ hostname_prefix }}- + count: 1 + groups: + - cloudera_manager + - cluster_masters + - cluster + - nodes + - db_server + - linux_nodes + - deployment + host_template: Masters + instance_type: m5.4xlarge + root_volume: + volume_size: 250 + subnet_index: 1 + tls: True + # Cluster Worker + - hostname_prefix: worker + count: 2 + groups: + - cluster_workers + - cluster + - nodes + - linux_nodes + - deployment + instance_type: c5.2xlarge + subnet_index: 1 + root_volume: + volume_size: 250 + host_template: Workers + tls: True diff --git a/private-cloud/pvc-base-on-aws/execution-environment.yml b/private-cloud/pvc-base-on-aws/execution-environment.yml new file mode 100644 index 0000000..dfb823a --- /dev/null +++ b/private-cloud/pvc-base-on-aws/execution-environment.yml @@ -0,0 +1,18 @@ +--- + +version: 3 + +images: + base_image: + name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections + +dependencies: + galaxy: + collections: + - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git + type: git + version: devel + - name: freeipa.ansible_freeipa + version: 1.11.0 + python: + - dnspython diff --git a/private-cloud/pvc-base-on-aws/external_setup.yml b/private-cloud/pvc-base-on-aws/external_setup.yml new file mode 100644 index 0000000..cdbe151 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/external_setup.yml @@ -0,0 +1,48 @@ +--- + +- name: Set up CDP Private Cloud external prerequisites + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up external prerequisites for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_ext.yml + +# NOTE: Cluster hosts become unavilable after krb5 client setup +# This is a temporary workaround while we debug the source of the issue +- name: Reboot all cluster hosts + hosts: cluster + gather_facts: no + become: yes + tasks: + - name: Reboot all cluster hosts + ansible.builtin.reboot: \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/group_vars/all.yml b/private-cloud/pvc-base-on-aws/group_vars/all.yml new file mode 100644 index 0000000..19f3184 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/group_vars/all.yml @@ -0,0 +1,3 @@ +--- +# If needed, set the following variable to tell Ansible which key to use +# ansible_ssh_private_key_file: ~/.ssh/some-private-key diff --git a/private-cloud/pvc-base-on-aws/hostvars.j2 b/private-cloud/pvc-base-on-aws/hostvars.j2 new file mode 100644 index 0000000..bf2ca91 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/hostvars.j2 @@ -0,0 +1,9 @@ +{# Collect and output individual host variables #} +{% macro host_variables(host) %} +{% set fields = [] %} +{% set _ = fields.append("ansible_user=" + host['ansible_user']) if 'ansible_user' in host %} +{% set _ = fields.append("host_template=" + host['host_template']) if 'host_template' in host %} +{% set _ = fields.append("label=" + host['label']) if 'label' in host %} +{% set _ = fields.append("tls=" + host['tls'] | string) if 'tls' in host %} +{{ host['inventory_hostname'] }} {{ fields | join(' ') }} +{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/instance_vars.j2 b/private-cloud/pvc-base-on-aws/instance_vars.j2 new file mode 100644 index 0000000..6e7063f --- /dev/null +++ b/private-cloud/pvc-base-on-aws/instance_vars.j2 @@ -0,0 +1,13 @@ +{# Define the metadata tags for the individual Openstack instances #} +{# Output should be TF map _entries_, not a map itself #} + +{% macro instance_tags(host) %} +{% set tags = {} %} +{% set _ = tags.update({ 'ansible_user': host.ansible_user }) if host.ansible_user is defined %} +{% set _ = tags.update({ 'host_template': host.host_template }) if host.host_template is defined %} +{% set _ = tags.update({ 'groups': host.groups | join(', ') }) if host.groups is defined %} +{% set _ = tags.update({ 'tls': host.tls | string }) if host.tls is defined %} +{% for k, v in tags.items() %} + {{ k }} = "{{ v }}"{{ "," if not loop.last else "" }} +{% endfor %} +{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/internal_setup.yml b/private-cloud/pvc-base-on-aws/internal_setup.yml new file mode 100644 index 0000000..18b56ea --- /dev/null +++ b/private-cloud/pvc-base-on-aws/internal_setup.yml @@ -0,0 +1,38 @@ +--- + +- name: Set up CDP Private Cloud internal prerequisites + hosts: localhost + connection: local + gather_facts: yes + vars: + definition_path: "./" + tasks: + - name: Set of deployment variables from definition.yml + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Prepare inventory for PvC Plays + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + tasks_from: prep_pvc.yml + tags: + - always + +- name: Init run tasks for all nodes + hosts: all + gather_facts: no + tasks: + - name: Group hosts by host template and TLS + ansible.builtin.include_role: + name: cloudera.cluster.deployment.groupby + + - name: Check connectivity to Inventory + ansible.builtin.wait_for_connection: + tags: + - always + +- name: Set up internal prerequisites for CDP Private Cloud + ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_int.yml diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/pvc-base-on-aws/pre_setup.yml new file mode 100644 index 0000000..fcf0bea --- /dev/null +++ b/private-cloud/pvc-base-on-aws/pre_setup.yml @@ -0,0 +1,233 @@ +--- + +# Run variable checker + +# Provision the infrastructure at the cloud provider +- name: Provision infrastructure resources + hosts: localhost + connection: local + gather_facts: yes + module_defaults: + # Set your definition-specific inventory host variables here. + # NOTE: the 'node' variable is in scope and contains each entry coming from + # the Terraform 'nodes' output variable. + ansible.builtin.add_host: + host_template: "{{ node.metadata.host_template | default(omit) }}" + tls: "{{ node.metadata.tls | default(omit) }}" + # TODO Review the SSH args, these are overwritten + ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -C -o ControlMaster=auto -o ControlPersist=1200s -o BatchMode=yes' + tags: infra + tasks: + - name: Provision the infrastructure resources in the cloud provider + when: infra_type != 'static' + ansible.builtin.import_role: + name: cloudera.infra.provision + vars: + provision_provider: "{{ infra_type }}" + provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" + provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" + provision_terraform_parallelism: "{{ terraform.parallelism | default(omit) }}" + provision_state_storage: "{{ terraform.state.storage }}" + provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" + provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" + provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" + provision_name_prefix: "{{ name_prefix }}" + provision_domain_suffix: "{{ domain }}" + provision_ssh_keypair_label: "{{ ssh_keypair.label | default(public_key_id) }}" + provision_ssh_keypair_public_key: "{{ ssh_keypair.public_key | default(public_key_text) }}" + provision_owner_email: "{{ owner_email }}" + provision_openstack_network_name: "{{ infra[infra_type].network_name | default(omit) }}" + provision_openstack_default_image_name: "{{ infra[infra_type].default_image_name | default(omit) }}" + provision_openstack_default_availability_zone: "{{ infra[infra_type].default_availability_zone | default(omit) }}" + provision_aws_ec2_region: "{{ infra[infra_type].region | default(omit) }}" + provision_aws_ec2_vpc_enable_dns_support: "{{ infra[infra_type].enable_dns | default(omit) }}" + provision_aws_ec2_vpc_enable_dns_hostnames: "{{ infra[infra_type].enable_dns | default(omit) }}" + provision_aws_ec2_public_subnets: "{{ infra[infra_type].public_subnets | default(omit) }}" + provision_aws_ec2_private_subnets: "{{ infra[infra_type].private_subnets | default(omit) }}" + provision_aws_ec2_default_security_group_ingress: "{{ infra[infra_type].default_security_group.ingress | default(omit) }}" + provision_instances: "{{ infra[infra_type].nodes | default([]) }}" + provision_tags: "{{ deployment_tags | default(omit) }}" + +# Confirm the availablity of all nodes +- name: Ensure node readiness + hosts: nodes + gather_facts: no + tasks: + - name: Ensure the node is reachable + ansible.builtin.wait_for_connection: + timeout: 60 + +# Prepare and mount any attached volumes +- name: Prepare and mount storage volumes + hosts: nodes + gather_facts: no + become: True + tasks: + - name: Prepare storage volumes + when: infra_type != "static" and storage_volumes | length > 0 + ansible.builtin.import_role: + name: cloudera.infra.mount + vars: + mount_volumes: "{{ storage_volumes }}" + mount_provider: "{{ infra_type }}" + +# Prepare the FreeIPA sidecar services +- name: Provision FreeIPA services + hosts: freeipa + gather_facts: yes + become: yes + vars: + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + # If not set specifically: if AWS, then VPC CIDR + 2 (see https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) + resolv_nameservers: "{{ upstream_nameservers | default(default_dns[infra_type]) }}" + default_dns: + aws: "{{ [ hostvars.localhost.provision.vpc.cidr | ansible.utils.ipmath(2) ] }}" + module_defaults: + freeipa.ansible_freeipa.ipadnszone: + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + freeipa.ansible_freeipa.ipadnsrecord: + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + tasks: + - name: Set up the FreeIPA server + ansible.builtin.import_role: + # name: cloudera.exe.freeipa_server + name: cloudera.infra.freeipa_server + vars: + ipaserver_hostname: "{{ inventory_hostname }}" + ipaserver_realm: "{{ freeipa.realm }}" + ipaserver_domain: "{{ name_prefix }}.{{ domain }}" + ipaserver_no_host_dns: yes + ipaserver_setup_firewalld: no + ipaserver_setup_dns: "{{ enable_dns }}" + ipaserver_resolv_nameservers: "{{ resolv_nameservers }}" + ipaserver_auto_reverse: "{{ enable_dns | ternary(True, omit) }}" + ipaserver_no_forwarders: "{{ enable_dns | ternary(True, omit) }}" + ipaserver_forward_policy: "{{ enable_dns | ternary('only', omit) }}" + ipaserver_recursion_acl_cidr: "{{ enable_dns | ternary(hostvars.localhost.provision.vpc.cidr, omit) }}" + ipaserver_copy_csr_to_controller: yes + ipaclient_mkhomedir: yes + ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" # TODO Add test for these parameters - error in role is opaque + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + + - name: Configure provisioned FreeIPA global DNS + when: enable_dns + block: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in provisioned FreeIPA service + freeipa.ansible_freeipa.ipadnszone: + zone_name: "{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + forward_policy: none + + - name: Create reverse DNS zone for '{{ hostvars.localhost.provision.vpc.cidr }}' + freeipa.ansible_freeipa.ipadnszone: + name_from_ip: "{{ hostvars.localhost.provision.vpc.cidr }}" + dynamic_update: yes + allow_sync_ptr: yes + + - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' + when: "'cluster_ecs_masters' in groups" + block: + - name: Ensure DNS zone for ECS 'apps' + freeipa.ansible_freeipa.ipadnszone: + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + forward_policy: none + + - name: Ensure DNS wildcard records for ECS 'apps' and domain + freeipa.ansible_freeipa.ipadnsrecord: + name: "*" + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + record_type: "A" + record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" + create_reverse: no + + - name: Resolve provisioned FreeIPA DNS server addresses + ansible.builtin.set_fact: + dns_server_ips: "{{ dns_server_ips | default([]) | union([ ansible_default_ipv4['address'] ]) }}" + delegate_to: localhost + delegate_facts: true + +# Update existing FreeIPA DNS services +- name: Configure existing FreeIPA global DNS + hosts: localhost + connection: local + gather_facts: no + vars: + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + dns_server: "{{ freeipa.server | default(lookup('ansible.builtin.env', 'IPA_HOST', default='Undefined')) }}" + module_defaults: + community.general.ipa_dnszone: + ipa_host: "{{ dns_server }}" + ipa_user: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default='Undefined')) }}" + ipa_pass: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" + tasks: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in existing FreeIPA service + when: enable_dns and dns_server is defined and 'freeipa' not in groups + block: + - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' + community.general.ipa_dnszone: + validate_certs: no + dynamicupdate: yes + allowsyncptr: yes + zone_name: "{{ name_prefix }}.{{ domain }}" + state: present + + - name: Create reverse DNS zone for '{{ infra_cidr }}' + community.general.ipa_dnszone: + validate_certs: no + dynamicupdate: yes + allowsyncptr: yes + zone_name: "{{ infra_cidr | ansible.utils.ipaddr('revdns') }}" + state: present + + - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' + when: "'cluster_ecs_masters' in groups" + block: + - name: Ensure DNS zone for ECS 'apps' + community.general.ipa_dnszone: + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + validate_certs: no + + - name: Ensure DNS wildcard records for ECS 'apps' and domain + community.general.ipa_dnszone: + name: "*" + zone_name: "apps.{{ name_prefix }}.{{ domain }}" + record_type: "A" + record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" + validate_certs: no + + - name: Resolve existing FreeIPA DNS server addresses + ansible.builtin.set_fact: + dns_hosts: "{{ [ dns_server ] }}" + dns_server_ips: "{{ [ lookup('community.general.dig', dns_server ) ] }}" + +# Enroll deployment hosts with FreeIPA Kerberos and possibly with DNS +- name: Register hosts with FreeIPA + hosts: nodes:!freeipa + gather_facts: yes + become: True + tasks: + - name: Register host with FreeIPA services + when: krb5_kdc_type == "Red Hat IPA" + ansible.builtin.import_role: + # name: cloudera.exe.freeipa_client + name: cloudera.infra.freeipa_client + vars: + ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" + ipaserver_realm: "{{ freeipa.realm }}" + ipa_hosts: "{{ groups['freeipa'] | default(hostvars.localhost.dns_hosts) }}" + ipa_server_ips: "{{ hostvars.localhost.dns_server_ips }}" + ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" + ipaadmin_principal: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default=omit)) }}" + enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + +# Prepare SME Lab resources +- name: Prepare TLS and supporting services resources + import_playbook: pre_setup_resources.yml + tags: + - prereq + - infra \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml new file mode 100644 index 0000000..21d0321 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml @@ -0,0 +1,16 @@ +--- + # Playbook to setup additional resources for PvC Deployment + +- name: Ensure additional requirements are available + hosts: nodes + gather_facts: no + become: yes + tasks: + + - name: Install required packages + ansible.builtin.package: + name: + - tree + - jq + - zip + state: present diff --git a/private-cloud/pvc-base-on-aws/pre_teardown.yml b/private-cloud/pvc-base-on-aws/pre_teardown.yml new file mode 100644 index 0000000..68c014f --- /dev/null +++ b/private-cloud/pvc-base-on-aws/pre_teardown.yml @@ -0,0 +1,24 @@ +--- + +- name: Deprovision the infrastructure resources + hosts: localhost + connection: local + gather_facts: no + tags: + - teardown + - provision + tasks: + - name: Deprovision the infrastructure resources in the cloud provider + when: groups['all'] | length > 0 + ansible.builtin.import_role: + name: cloudera.infra.provision + vars: + provision_state: absent + provision_provider: "{{ infra_type }}" + provision_inventory_file: inventory_static.ini + provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" + provision_state_storage: "{{ terraform.state.storage }}" + provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" + provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" + provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" + provision_name_prefix: "{{ name_prefix }}" diff --git a/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml b/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml new file mode 100644 index 0000000..236169e --- /dev/null +++ b/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml @@ -0,0 +1,27 @@ +- name: Check that all hosts in inventory are reachable + hosts: all + gather_facts: no + tasks: + - name: Ping each host from controller + ping: + + +- name: Check Forward and Reverse DNS lookups + hosts: localhost + connection: local + gather_facts: no + tasks: + + - name: Print the list of hosts + debug: + var: groups['all'] + + - name: Forward lookup + ansible.builtin.debug: + msg: + - "Forward lookup DNS for {{ item }} is {{ lookup('community.general.dig', item , qtype='A') }}" + loop: "{{ groups['all'] }}" + + # TODO Register forward lookup IP and do reverse lookup + + From 67494ad73cc64ab9b67e628254822b5942eb7d80 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:30:59 -0500 Subject: [PATCH 05/45] Updates for 7.1.9 cluster Signed-off-by: Jim Enright --- .../ecs-on-aws/ansible-navigator.yml | 7 ++++++- private-cloud/ecs-on-aws/cluster.yml | 6 +++++- private-cloud/ecs-on-aws/definition.yml | 17 +++++++--------- .../ecs-on-aws/execution-environment.yml | 8 +++----- private-cloud/ecs-on-aws/pre_setup.yml | 10 ++++------ .../ecs-on-aws/pre_setup_resources.yml | 8 ++++++++ private-cloud/ecs-on-aws/pre_teardown.yml | 2 +- .../pvc-base-on-aws/ansible-navigator.yml | 7 ++++++- private-cloud/pvc-base-on-aws/cluster.yml | 5 +++++ private-cloud/pvc-base-on-aws/definition.yml | 19 ++++++++---------- .../pvc-base-on-aws/execution-environment.yml | 20 ++++++++++++------- private-cloud/pvc-base-on-aws/pre_setup.yml | 10 ++++------ .../pvc-base-on-aws/pre_setup_resources.yml | 7 +++++++ .../pvc-base-on-aws/pre_teardown.yml | 2 +- 14 files changed, 78 insertions(+), 50 deletions(-) diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml index e1c62c0..83ac430 100644 --- a/private-cloud/ecs-on-aws/ansible-navigator.yml +++ b/private-cloud/ecs-on-aws/ansible-navigator.yml @@ -30,11 +30,16 @@ ansible-navigator: ANSIBLE_DEPRECATION_WARNINGS: False ANSIBLE_HOST_KEY_CHECKING: False ANSIBLE_SSH_RETRIES: 10 - image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/" + # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + image: ghcr.io/cloudera-labs/cldr-runner:aws-latest pull: arguments: - "--tls-verify=false" volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" - src: "${SSH_PUBLIC_KEY_FILE}" dest: "${SSH_PUBLIC_KEY_FILE}" - src: "${CDP_LICENSE_FILE}" diff --git a/private-cloud/ecs-on-aws/cluster.yml b/private-cloud/ecs-on-aws/cluster.yml index 85cdecb..622afaa 100644 --- a/private-cloud/ecs-on-aws/cluster.yml +++ b/private-cloud/ecs-on-aws/cluster.yml @@ -28,7 +28,6 @@ clusters: longhorn_replication: 2 lsoDataPath: "/ecs/local" defaultDataPath: "/ecs/longhorn-storage" - nfs_provisioned: 800 nfs_over_provisioning: 800 # TODO: Create cert and key for ECS ingress controller # ssl_certificate: /opt/cloudera/security/pki/ecs.pem @@ -143,6 +142,11 @@ clusters: STORAGE_CONTAINER_MANAGER: scm_max_heap_size: 2048 OZONE_DATANODE: + ozone-conf/ozone-site.xml_role_safety_valve: | + + hdds.datanode.client.port + 9874 + ozone_datanode_heap_size: 2048 OZONE_PROMETHEUS: ozone.prometheus.http-port: 19090 diff --git a/private-cloud/ecs-on-aws/definition.yml b/private-cloud/ecs-on-aws/definition.yml index a9c6d6a..9a04533 100644 --- a/private-cloud/ecs-on-aws/definition.yml +++ b/private-cloud/ecs-on-aws/definition.yml @@ -6,9 +6,9 @@ encryption_activated: True # autotls: False repositories: - # 7.1.8 CHF4 - - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/ -cloudera_manager_version: 7.10.1 + # Offical CDH 7.1.9.0 + - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/ +cloudera_manager_version: 7.11.3 jdk_version: 11 @@ -40,6 +40,8 @@ dns_provider: "freeipa" # Connect FreeIPA to Knox and Ranger freeipa_activated: yes +# FreeIPA client install on cluster nodes is done during pre-setup +freeipa_enroll: no # FreeIPA realm settings freeipa: @@ -64,12 +66,6 @@ cloudera_manager_options: CUSTOM_BANNER_HTML: "1.5.1 - PvC ECS Lab ({{ name_prefix }})" SESSION_TIMEOUT: 43200 PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 - # NOTE: Disabled in ChuckL's latest definition - # PHONE_HOME: false - # PROXYUSER_KNOX_GROUPS: "*" - # PROXYUSER_KNOX_HOSTS: "*" - # PROXYUSER_KNOX_PRINCIPAL: "knox" - # PROXYUSER_KNOX_USERS: "*" KRB_AUTH_ENABLE: "true" # License options (this is due to a hardcoded tmp directory on the target/manager node) @@ -81,8 +77,9 @@ license_local_tmp_path: /tmp/cloudera_license.txt database_type: postgresql database_version: 12 database_tls: true +database_default_password: "{{ common_password }}" -#########++++ +############### ## Red Hat FreeIPA krb5_kdc_type: Red Hat IPA diff --git a/private-cloud/ecs-on-aws/execution-environment.yml b/private-cloud/ecs-on-aws/execution-environment.yml index dfb823a..a097142 100644 --- a/private-cloud/ecs-on-aws/execution-environment.yml +++ b/private-cloud/ecs-on-aws/execution-environment.yml @@ -4,15 +4,13 @@ version: 3 images: base_image: - name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections + name: ghcr.io/cloudera-labs/cldr-runner:aws-latest dependencies: galaxy: collections: - - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git + - name: https://github.com/jimright/cloudera.exe.git type: git - version: devel - - name: freeipa.ansible_freeipa - version: 1.11.0 + version: feature/tmp_pvc python: - dnspython diff --git a/private-cloud/ecs-on-aws/pre_setup.yml b/private-cloud/ecs-on-aws/pre_setup.yml index fcf0bea..b4035c5 100644 --- a/private-cloud/ecs-on-aws/pre_setup.yml +++ b/private-cloud/ecs-on-aws/pre_setup.yml @@ -21,7 +21,7 @@ - name: Provision the infrastructure resources in the cloud provider when: infra_type != 'static' ansible.builtin.import_role: - name: cloudera.infra.provision + name: cloudera.exe.provision vars: provision_provider: "{{ infra_type }}" provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" @@ -66,7 +66,7 @@ - name: Prepare storage volumes when: infra_type != "static" and storage_volumes | length > 0 ansible.builtin.import_role: - name: cloudera.infra.mount + name: cloudera.exe.mount vars: mount_volumes: "{{ storage_volumes }}" mount_provider: "{{ infra_type }}" @@ -90,8 +90,7 @@ tasks: - name: Set up the FreeIPA server ansible.builtin.import_role: - # name: cloudera.exe.freeipa_server - name: cloudera.infra.freeipa_server + name: cloudera.exe.freeipa_server vars: ipaserver_hostname: "{{ inventory_hostname }}" ipaserver_realm: "{{ freeipa.realm }}" @@ -214,8 +213,7 @@ - name: Register host with FreeIPA services when: krb5_kdc_type == "Red Hat IPA" ansible.builtin.import_role: - # name: cloudera.exe.freeipa_client - name: cloudera.infra.freeipa_client + name: cloudera.exe.freeipa_client vars: ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" ipaserver_realm: "{{ freeipa.realm }}" diff --git a/private-cloud/ecs-on-aws/pre_setup_resources.yml b/private-cloud/ecs-on-aws/pre_setup_resources.yml index 21d0321..e4270be 100644 --- a/private-cloud/ecs-on-aws/pre_setup_resources.yml +++ b/private-cloud/ecs-on-aws/pre_setup_resources.yml @@ -13,4 +13,12 @@ - tree - jq - zip + - python38 state: present + + - name: Install required packages + ansible.builtin.pip: + name: psycopg2-binary==2.9.5 + executable: pip3.8 + state: present + diff --git a/private-cloud/ecs-on-aws/pre_teardown.yml b/private-cloud/ecs-on-aws/pre_teardown.yml index 68c014f..7ceedf0 100644 --- a/private-cloud/ecs-on-aws/pre_teardown.yml +++ b/private-cloud/ecs-on-aws/pre_teardown.yml @@ -11,7 +11,7 @@ - name: Deprovision the infrastructure resources in the cloud provider when: groups['all'] | length > 0 ansible.builtin.import_role: - name: cloudera.infra.provision + name: cloudera.exe.provision vars: provision_state: absent provision_provider: "{{ infra_type }}" diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml index e1c62c0..83ac430 100644 --- a/private-cloud/pvc-base-on-aws/ansible-navigator.yml +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -30,11 +30,16 @@ ansible-navigator: ANSIBLE_DEPRECATION_WARNINGS: False ANSIBLE_HOST_KEY_CHECKING: False ANSIBLE_SSH_RETRIES: 10 - image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/" + # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections + image: ghcr.io/cloudera-labs/cldr-runner:aws-latest pull: arguments: - "--tls-verify=false" volume-mounts: + - src: "${ANSIBLE_COLLECTIONS_PATH}" + dest: "${ANSIBLE_COLLECTIONS_PATH}" + options: "Z" - src: "${SSH_PUBLIC_KEY_FILE}" dest: "${SSH_PUBLIC_KEY_FILE}" - src: "${CDP_LICENSE_FILE}" diff --git a/private-cloud/pvc-base-on-aws/cluster.yml b/private-cloud/pvc-base-on-aws/cluster.yml index a59f68b..5e600bd 100644 --- a/private-cloud/pvc-base-on-aws/cluster.yml +++ b/private-cloud/pvc-base-on-aws/cluster.yml @@ -96,6 +96,11 @@ clusters: STORAGE_CONTAINER_MANAGER: scm_max_heap_size: 2048 OZONE_DATANODE: + ozone-conf/ozone-site.xml_role_safety_valve: | + + hdds.datanode.client.port + 9874 + ozone_datanode_heap_size: 2048 OZONE_PROMETHEUS: ozone.prometheus.http-port: 19090 diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/pvc-base-on-aws/definition.yml index 478b1d1..8405c6f 100644 --- a/private-cloud/pvc-base-on-aws/definition.yml +++ b/private-cloud/pvc-base-on-aws/definition.yml @@ -6,9 +6,9 @@ encryption_activated: True # autotls: False repositories: - # 7.1.8 CHF4 - - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/ -cloudera_manager_version: 7.10.1 + # Offical CDH 7.1.9.0 + - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/ +cloudera_manager_version: 7.11.3 jdk_version: 11 @@ -33,6 +33,8 @@ dns_provider: "freeipa" # Connect FreeIPA to Knox and Ranger freeipa_activated: yes +# FreeIPA client install on cluster nodes is done during pre-setup +freeipa_enroll: no # FreeIPA realm settings freeipa: @@ -54,15 +56,9 @@ skip_user_group_init: no # Cloudera Manager details cloudera_manager_options: - CUSTOM_BANNER_HTML: "1.5.1 - PvC Base Lab ({{ name_prefix }})" + CUSTOM_BANNER_HTML: "PvC Base Lab ({{ name_prefix }})" SESSION_TIMEOUT: 43200 PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 - # NOTE: Disabled in ChuckL's latest definition - # PHONE_HOME: false - # PROXYUSER_KNOX_GROUPS: "*" - # PROXYUSER_KNOX_HOSTS: "*" - # PROXYUSER_KNOX_PRINCIPAL: "knox" - # PROXYUSER_KNOX_USERS: "*" KRB_AUTH_ENABLE: "true" # License options (this is due to a hardcoded tmp directory on the target/manager node) @@ -74,8 +70,9 @@ license_local_tmp_path: /tmp/cloudera_license.txt database_type: postgresql database_version: 12 database_tls: true +database_default_password: "{{ common_password }}" -#########++++ +############### ## Red Hat FreeIPA krb5_kdc_type: Red Hat IPA diff --git a/private-cloud/pvc-base-on-aws/execution-environment.yml b/private-cloud/pvc-base-on-aws/execution-environment.yml index dfb823a..e10bc84 100644 --- a/private-cloud/pvc-base-on-aws/execution-environment.yml +++ b/private-cloud/pvc-base-on-aws/execution-environment.yml @@ -4,15 +4,21 @@ version: 3 images: base_image: - name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections + name: ghcr.io/cloudera-labs/cldr-runner:aws-latest dependencies: galaxy: collections: - - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git + - name: https://github.com/jimright/cloudera.exe.git type: git - version: devel - - name: freeipa.ansible_freeipa - version: 1.11.0 - python: - - dnspython + version: feature/tmp_pvc + # - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git + # type: git + # version: devel + # - name: https://github.com/anisf/cloudera.cluster.git + # type: git + # version: "fix/#126" + # - name: freeipa.ansible_freeipa + # version: 1.11.0 + # python: + # - dnspython diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/pvc-base-on-aws/pre_setup.yml index fcf0bea..b4035c5 100644 --- a/private-cloud/pvc-base-on-aws/pre_setup.yml +++ b/private-cloud/pvc-base-on-aws/pre_setup.yml @@ -21,7 +21,7 @@ - name: Provision the infrastructure resources in the cloud provider when: infra_type != 'static' ansible.builtin.import_role: - name: cloudera.infra.provision + name: cloudera.exe.provision vars: provision_provider: "{{ infra_type }}" provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" @@ -66,7 +66,7 @@ - name: Prepare storage volumes when: infra_type != "static" and storage_volumes | length > 0 ansible.builtin.import_role: - name: cloudera.infra.mount + name: cloudera.exe.mount vars: mount_volumes: "{{ storage_volumes }}" mount_provider: "{{ infra_type }}" @@ -90,8 +90,7 @@ tasks: - name: Set up the FreeIPA server ansible.builtin.import_role: - # name: cloudera.exe.freeipa_server - name: cloudera.infra.freeipa_server + name: cloudera.exe.freeipa_server vars: ipaserver_hostname: "{{ inventory_hostname }}" ipaserver_realm: "{{ freeipa.realm }}" @@ -214,8 +213,7 @@ - name: Register host with FreeIPA services when: krb5_kdc_type == "Red Hat IPA" ansible.builtin.import_role: - # name: cloudera.exe.freeipa_client - name: cloudera.infra.freeipa_client + name: cloudera.exe.freeipa_client vars: ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" ipaserver_realm: "{{ freeipa.realm }}" diff --git a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml index 21d0321..91f66b8 100644 --- a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml +++ b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml @@ -13,4 +13,11 @@ - tree - jq - zip + - python38 + state: present + + - name: Install required packages + ansible.builtin.pip: + name: psycopg2-binary==2.9.5 + executable: pip3.8 state: present diff --git a/private-cloud/pvc-base-on-aws/pre_teardown.yml b/private-cloud/pvc-base-on-aws/pre_teardown.yml index 68c014f..7ceedf0 100644 --- a/private-cloud/pvc-base-on-aws/pre_teardown.yml +++ b/private-cloud/pvc-base-on-aws/pre_teardown.yml @@ -11,7 +11,7 @@ - name: Deprovision the infrastructure resources in the cloud provider when: groups['all'] | length > 0 ansible.builtin.import_role: - name: cloudera.infra.provision + name: cloudera.exe.provision vars: provision_state: absent provision_provider: "{{ infra_type }}" From 7a923d6e6a67e3c9e8e393d6641d4a49c11ad977 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:31:00 -0500 Subject: [PATCH 06/45] Update PvC examples to use latest collection release Signed-off-by: Jim Enright --- private-cloud/ecs-on-aws/README.md | 51 ++++--------------- .../ecs-on-aws/ansible-navigator.yml | 5 -- .../ecs-on-aws/execution-environment.yml | 16 ------ private-cloud/pvc-base-on-aws/README.md | 51 ++++--------------- .../pvc-base-on-aws/ansible-navigator.yml | 5 -- .../pvc-base-on-aws/execution-environment.yml | 24 --------- 6 files changed, 18 insertions(+), 134 deletions(-) delete mode 100644 private-cloud/ecs-on-aws/execution-environment.yml delete mode 100644 private-cloud/pvc-base-on-aws/execution-environment.yml diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md index df7ec25..3242293 100644 --- a/private-cloud/ecs-on-aws/README.md +++ b/private-cloud/ecs-on-aws/README.md @@ -2,51 +2,19 @@ > Constructs a CDP Private Cloud Base cluster suitable for enablement in configuring and using Ozone on a Private Cloud ECS Data Services (DS) cluster. -## Requirements - -This example requires an execution environment with dependencies to run the automation; and a set of configuration variables. - -We provide instructions for using a container based execution environment. - -### Container Execution Environment - -1. Create and activate a new `virtualenv` and install `ansible-core` and `ansible-navigator` - - ```bash - python -m venv ~/cdp-navigator; - - source ~/cdp-navigator/bin/activate; - - pip install ansible-core==2.12.10 ansible-navigator==3.4.0 - ``` - -1. Clone this repository. +## Known Issues - ```bash - git clone https://github.com/cloudera-labs/cloudera-deploy.git; - ``` +| Issue | Description | Workaround | +|-------|-------------|------------| +| Cluster instances unavailable after the `external_setup.yml` Playbook | The cluster EC2 instances become unavailable after the `external_setup.yml` Playbook. During subsequent playbooks the hosts becomes unreachable and in the EC2 console the VM instances fail the reachability health check. | Restart the EC2 instances via the console. | -1. Change your working directory to this project. - - ```bash - cd cloudera-deploy/private-cloud/ecs-on-aws - ``` - -1. We currently need to build a local `cldr-runner` image for use as an Execution Environment. _(This is necessary at this point in the release cycle. It will become optional.)_ - - ```bash - ansible-navigator builder build --prune-images -v 3 --tag ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections - ``` - -This step sometimes takes a ~~number of minutes~~ long time to complete. YMMV. - -Once constructed, you can check the image by running `ansible-navigator` and using the prompts to examine: +## Requirements -```bash -ansible-navigator images -``` +To run, you need: -Or by running the Docker command directly, `docker image ls`. +* Docker (or a Docker alternative) +* AWS credentials (set via `AWS_PROFILE`) +* CDP Private Cloud Base license file credentials (set via `CDP_LICENSE_FILE`) ### Configuration Variables @@ -66,7 +34,6 @@ Configuration is passed via environment variables and an user-facing configurati > **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. - #### Configuration file variables Edit the `config.yml` user-facing configuration file to match your particular deployment. diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml index 83ac430..a83766b 100644 --- a/private-cloud/ecs-on-aws/ansible-navigator.yml +++ b/private-cloud/ecs-on-aws/ansible-navigator.yml @@ -30,16 +30,11 @@ ansible-navigator: ANSIBLE_DEPRECATION_WARNINGS: False ANSIBLE_HOST_KEY_CHECKING: False ANSIBLE_SSH_RETRIES: 10 - ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/" - # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections image: ghcr.io/cloudera-labs/cldr-runner:aws-latest pull: arguments: - "--tls-verify=false" volume-mounts: - - src: "${ANSIBLE_COLLECTIONS_PATH}" - dest: "${ANSIBLE_COLLECTIONS_PATH}" - options: "Z" - src: "${SSH_PUBLIC_KEY_FILE}" dest: "${SSH_PUBLIC_KEY_FILE}" - src: "${CDP_LICENSE_FILE}" diff --git a/private-cloud/ecs-on-aws/execution-environment.yml b/private-cloud/ecs-on-aws/execution-environment.yml deleted file mode 100644 index a097142..0000000 --- a/private-cloud/ecs-on-aws/execution-environment.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -version: 3 - -images: - base_image: - name: ghcr.io/cloudera-labs/cldr-runner:aws-latest - -dependencies: - galaxy: - collections: - - name: https://github.com/jimright/cloudera.exe.git - type: git - version: feature/tmp_pvc - python: - - dnspython diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/pvc-base-on-aws/README.md index 1e4bebf..8618d71 100644 --- a/private-cloud/pvc-base-on-aws/README.md +++ b/private-cloud/pvc-base-on-aws/README.md @@ -2,51 +2,19 @@ > Constructs a CDP Private Cloud Base cluster running on AWS. -## Requirements - -This example requires an execution environment with dependencies to run the automation; and a set of configuration variables. - -We provide instructions for using a container based execution environment. - -### Container Execution Environment - -1. Create and activate a new `virtualenv` and install `ansible-core` and `ansible-navigator` - - ```bash - python -m venv ~/cdp-navigator; - - source ~/cdp-navigator/bin/activate; - - pip install ansible-core==2.12.10 ansible-navigator==3.4.0 - ``` - -1. Clone this repository. +## Known Issues - ```bash - git clone https://github.com/cloudera-labs/cloudera-deploy.git; - ``` +| Issue | Description | Workaround | +|-------|-------------|------------| +| Cluster instances unavailable after the `external_setup.yml` Playbook | The cluster EC2 instances become unavailable after the `external_setup.yml` Playbook. During subsequent playbooks the hosts becomes unreachable and in the EC2 console the VM instances fail the reachability health check. | Restart the EC2 instances via the console. | -1. Change your working directory to this project. - - ```bash - cd cloudera-deploy/private-cloud/pvc-base-on-aws - ``` - -1. We currently need to build a local `cldr-runner` image for use as an Execution Environment. _(This is necessary at this point in the release cycle. It will become optional.)_ - - ```bash - ansible-navigator builder build --prune-images -v 3 --tag ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections - ``` - -This step sometimes takes a ~~number of minutes~~ long time to complete. YMMV. - -Once constructed, you can check the image by running `ansible-navigator` and using the prompts to examine: +## Requirements -```bash -ansible-navigator images -``` +To run, you need: -Or by running the Docker command directly, `docker image ls`. +* Docker (or a Docker alternative) +* AWS credentials (set via `AWS_PROFILE`) +* CDP Private Cloud Base license file credentials (set via `CDP_LICENSE_FILE`) ### Configuration Variables @@ -66,7 +34,6 @@ Configuration is passed via environment variables and an user-facing configurati > **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. - #### Configuration file variables Edit the `config.yml` user-facing configuration file to match your particular deployment. diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml index 83ac430..a83766b 100644 --- a/private-cloud/pvc-base-on-aws/ansible-navigator.yml +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -30,16 +30,11 @@ ansible-navigator: ANSIBLE_DEPRECATION_WARNINGS: False ANSIBLE_HOST_KEY_CHECKING: False ANSIBLE_SSH_RETRIES: 10 - ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/" - # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections image: ghcr.io/cloudera-labs/cldr-runner:aws-latest pull: arguments: - "--tls-verify=false" volume-mounts: - - src: "${ANSIBLE_COLLECTIONS_PATH}" - dest: "${ANSIBLE_COLLECTIONS_PATH}" - options: "Z" - src: "${SSH_PUBLIC_KEY_FILE}" dest: "${SSH_PUBLIC_KEY_FILE}" - src: "${CDP_LICENSE_FILE}" diff --git a/private-cloud/pvc-base-on-aws/execution-environment.yml b/private-cloud/pvc-base-on-aws/execution-environment.yml deleted file mode 100644 index e10bc84..0000000 --- a/private-cloud/pvc-base-on-aws/execution-environment.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -version: 3 - -images: - base_image: - name: ghcr.io/cloudera-labs/cldr-runner:aws-latest - -dependencies: - galaxy: - collections: - - name: https://github.com/jimright/cloudera.exe.git - type: git - version: feature/tmp_pvc - # - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git - # type: git - # version: devel - # - name: https://github.com/anisf/cloudera.cluster.git - # type: git - # version: "fix/#126" - # - name: freeipa.ansible_freeipa - # version: 1.11.0 - # python: - # - dnspython From 567ee49eb87222c025449ba3177806be6e67a6fb Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 7 Dec 2023 15:31:00 -0500 Subject: [PATCH 07/45] Update requirements for length of name_prefix Signed-off-by: Jim Enright --- private-cloud/ecs-on-aws/README.md | 2 +- private-cloud/pvc-base-on-aws/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md index 3242293..0b8c7b5 100644 --- a/private-cloud/ecs-on-aws/README.md +++ b/private-cloud/ecs-on-aws/README.md @@ -38,7 +38,7 @@ Configuration is passed via environment variables and an user-facing configurati Edit the `config.yml` user-facing configuration file to match your particular deployment. -*NOTE:* `name_prefix` should be 4-8 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. +*NOTE:* `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. ```yaml name_prefix: "labaw" # CHANGE THIS diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/pvc-base-on-aws/README.md index 8618d71..37d7ad7 100644 --- a/private-cloud/pvc-base-on-aws/README.md +++ b/private-cloud/pvc-base-on-aws/README.md @@ -38,7 +38,7 @@ Configuration is passed via environment variables and an user-facing configurati Edit the `config.yml` user-facing configuration file to match your particular deployment. -*NOTE:* `name_prefix` should be 4-8 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. +*NOTE:* `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. ```yaml name_prefix: "labaw" # CHANGE THIS From c2a5ac4e5bbec9959d3d6306ff3e3c33be454d02 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:00 -0500 Subject: [PATCH 08/45] Add Terraform module for bastion host Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_bastion/main.tf | 107 ++++++++++++++++++ .../pvc-base-on-aws/tf_bastion/outputs.tf | 16 +++ .../pvc-base-on-aws/tf_bastion/variables.tf | 35 ++++++ 3 files changed, 158 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/tf_bastion/main.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_bastion/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf new file mode 100644 index 0000000..c810f46 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf @@ -0,0 +1,107 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws", + version = ">= 4.60.0", + } + ansible = { + source = "ansible/ansible" + version = ">= 1.0.0" + } + } +} + +provider "aws" { + region = var.region + default_tags { + tags = var.asset_tags + } +} + +data "local_file" "ssh_public_key_file" { + filename = var.ssh_public_key_file +} + +resource "aws_key_pair" "deployment" { + key_name = "${var.prefix}-private-cloud-base-bastion" + public_key = data.local_file.ssh_public_key_file.content +} + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +resource "aws_security_group" "bastion" { + name = "${var.prefix}-private-cloud-base-bastion" + description = "Allow SSH traffic" + vpc_id = var.vpc_id +} + +resource "aws_vpc_security_group_egress_rule" "bastion" { + security_group_id = aws_security_group.bastion.id + + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = -1 + description = "All traffic" +} + +resource "aws_vpc_security_group_ingress_rule" "bastion" { + security_group_id = aws_security_group.bastion.id + + cidr_ipv4 = "0.0.0.0/0" + from_port = 22 + ip_protocol = "tcp" + to_port = 22 + description = "All SSH traffic" +} + +resource "aws_instance" "bastion" { + ami = data.aws_ami.ubuntu.id + instance_type = "t2.micro" + key_name = aws_key_pair.deployment.key_name + subnet_id = var.subnet_id + vpc_security_group_ids = [ + aws_security_group.bastion.id, + ] + + associate_public_ip_address = true + + tags = { + Name = "${var.prefix}-private-cloud-base-bastion" + } +} + +resource "ansible_host" "bastion" { + name = aws_instance.bastion.public_dns + groups = ["bastion"] + variables = { + ansible_user = "ubuntu" + } +} diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf new file mode 100644 index 0000000..7abb9d8 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf @@ -0,0 +1,16 @@ +output "ssh_key_pair" { + value = { + name = aws_key_pair.deployment.key_name + public_key = data.local_file.ssh_public_key_file + fingerprint = aws_key_pair.deployment.fingerprint + } + description = "CDP Private Cloud bastion" +} + +output "instance" { + value = aws_instance.bastion +} + +output "security_group" { + value = aws_security_group.bastion +} diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf new file mode 100644 index 0000000..e80a249 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf @@ -0,0 +1,35 @@ +variable "asset_tags" { + type = map(any) + default = {} + description = "Map of tags applied to all cloud-provider assets." +} + +variable "ssh_public_key_file" { + type = string + description = "Local SSH public key file" +} + +variable "prefix" { + type = string + description = "Deployment prefix for all cloud-provider assets." + + validation { + condition = length(var.prefix) < 8 || length(var.prefix) > 4 + error_message = "Valid length for prefix is between 4-7 characters." + } +} + +variable "region" { + type = string + description = "Target AWS region" +} + +variable "vpc_id" { + type = string + description = "Target VPC ID" +} + +variable "subnet_id" { + type = string + description = "Target Subnet ID" +} From 696c428a785edb7fbeba652fe97caf4384b3cdf0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:00 -0500 Subject: [PATCH 09/45] Add Terraform module for cluster assets Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_cluster/main.tf | 97 ++++++++ .../pvc-base-on-aws/tf_cluster/network.tf | 222 ++++++++++++++++++ .../pvc-base-on-aws/tf_cluster/variables.tf | 153 ++++++++++++ 3 files changed, 472 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/main.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/network.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_cluster/main.tf new file mode 100644 index 0000000..68db9c3 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_cluster/main.tf @@ -0,0 +1,97 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws", + version = ">= 4.60.0", + } + ansible = { + source = "ansible/ansible" + version = ">= 1.0.0" + } + } +} + +provider "aws" { + region = var.region + default_tags { + tags = var.asset_tags + } +} + +data "local_file" "ssh_public_key_file" { + filename = var.ssh_public_key_file +} + +resource "aws_key_pair" "deployment" { + key_name = "${var.prefix}-pvc-base" + public_key = data.local_file.ssh_public_key_file.content +} + +# data "aws_ami" "ubuntu" { +# most_recent = true + +# filter { +# name = "name" +# values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] +# } + +# filter { +# name = "virtualization-type" +# values = ["hvm"] +# } + +# owners = ["099720109477"] # Canonical +# } + +# resource "aws_security_group" "bastion" { +# name = "${var.prefix}-private-cloud-base-bastion" +# description = "Allow SSH traffic" +# vpc_id = var.vpc_id +# } + +# resource "aws_vpc_security_group_egress_rule" "cluster" { +# security_group_id = aws_security_group.bastion.id + +# cidr_ipv4 = "0.0.0.0/0" +# ip_protocol = -1 +# description = "All traffic" +# } + +# resource "aws_instance" "bastion" { +# ami = data.aws_ami.ubuntu.id +# instance_type = "t2.micro" +# key_name = aws_key_pair.deployment.key_name +# subnet_id = var.subnet_id +# vpc_security_group_ids = [ +# aws_security_group.bastion.id, +# ] + +# associate_public_ip_address = true + +# tags = { +# Name = "${var.prefix}-private-cloud-base-bastion" +# } +# } + +# resource "ansible_host" "bastion" { +# name = aws_instance.bastion.public_dns +# groups = ["bastion"] +# variables = { +# ansible_user = "ubuntu" +# } +# } diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf new file mode 100644 index 0000000..b2f1011 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf @@ -0,0 +1,222 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO Move VPC creation to tf_proxied_cluster root module + +data "aws_availability_zones" "pvc_base" { + state = "available" +} + +locals { + vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" + igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" + nat_name = var.nat_gateway_name != "" ? var.nat_gateway_name : "${var.prefix}-pvc-base-nat" + rt_public_name = var.public_route_table_name != "" ? var.public_route_table_name : "${var.prefix}-pvc-base-public" + rt_private_name = var.private_route_table_name != "" ? var.private_route_table_name : "${var.prefix}-pvc-base-private" + public_subnet_name = var.public_subnet_name != "" ? var.public_subnet_name : "${var.prefix}-pvc-base-public" + public_subnets = length(var.public_subnets) > 0 ? var.public_subnets : tolist([{ + name = "${local.public_subnet_name}-01", + cidr = cidrsubnet(var.vpc_cidr, 8, 0), + az = data.aws_availability_zones.pvc_base.names[0], + tags = {} + }]) + private_subnet_name = var.private_subnet_name != "" ? var.private_subnet_name : "${var.prefix}-pvc-base-private" + private_subnets = length(var.private_subnets) > 0 ? var.private_subnets : tolist([{ + name = "${local.private_subnet_name}-01", + cidr = cidrsubnet(var.vpc_cidr, 8, 1), + az = data.aws_availability_zones.pvc_base.names[0], + tags = {} + }]) +} + +# ------- AWS VPC ------- +resource "aws_vpc" "pvc_base" { + cidr_block = var.vpc_cidr + tags = { Name = local.vpc_name } + instance_tenancy = "default" + enable_dns_support = true + enable_dns_hostnames = true +} + +# Internet Gateway +resource "aws_internet_gateway" "pvc_base" { + vpc_id = aws_vpc.pvc_base.id + tags = { Name = local.igw_name } +} + +# ------- AWS Public Network infrastructure ------- + +# TODO Calculate a local for a single default public subnet or defer to the root tf_proxied_cluter module +# TODO Lookup AZ id by input (name) and if not found, use AZ value as ID + +# Public Subnets +resource "aws_subnet" "pvc_base_public" { + for_each = { for idx, subnet in local.public_subnets : idx => subnet } + + vpc_id = aws_vpc.pvc_base.id + cidr_block = each.value.cidr + map_public_ip_on_launch = true + availability_zone = each.value.az + tags = merge(each.value.tags, { Name = each.value.name }) +} + +resource "aws_route_table" "pvc_base_public" { + for_each = { for idx, subnet in local.public_subnets : idx => subnet } + + vpc_id = aws_vpc.pvc_base.id + + tags = { Name = format("%s-%02d", local.rt_public_name, index(local.public_subnets, each.value) + 1) } + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.pvc_base.id + } +} + +# Public Route Table Associations +resource "aws_route_table_association" "pvc_base_public" { + for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } + + subnet_id = each.value.id + route_table_id = aws_route_table.pvc_base_public[each.key].id +} + +# ------- AWS Private Networking infrastructure ------- +# Network Gateways (NAT) +resource "aws_eip" "pvc_base" { + count = length(aws_subnet.pvc_base_public) + + tags = { Name = format("%s-%02d", local.nat_name, count.index + 1) } +} + +resource "aws_nat_gateway" "pvc_base" { + for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } + + subnet_id = each.value.id + allocation_id = aws_eip.pvc_base[each.key].id + connectivity_type = "public" + tags = { Name = format("%s-%02d", local.nat_name, each.key + 1) } +} + +# Private Subnets +resource "aws_subnet" "pvc_base_private" { + for_each = { for idx, subnet in local.private_subnets : idx => subnet } + + vpc_id = aws_vpc.pvc_base.id + cidr_block = each.value.cidr + map_public_ip_on_launch = false + availability_zone = each.value.az + tags = merge(each.value.tags, { Name = each.value.name }) +} + +# Private Route Tables +resource "aws_route_table" "pvc_base_private" { + for_each = { for idx, subnet in local.private_subnets : idx => subnet } + + vpc_id = aws_vpc.pvc_base.id + + tags = { Name = format("%s-%02d", local.rt_private_name, index(local.private_subnets, each.value)) } + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.pvc_base[(index(local.private_subnets, each.value) % length(aws_nat_gateway.pvc_base))].id + } +} + +# Private Route Table Associations +resource "aws_route_table_association" "pvc_base_private" { + for_each = { for idx, subnet in aws_subnet.pvc_base_private : idx => subnet } + + subnet_id = each.value.id + route_table_id = aws_route_table.pvc_base_private[each.key].id +} + +# # ------- Security Groups ------- +# # Default SG +# resource "aws_security_group" "cdp_default_sg" { +# vpc_id = aws_vpc.pvc_base.id +# name = var.security_group_default_name +# description = var.security_group_default_name + +# tags = merge(var.env_tags, { Name = var.security_group_default_name }) + +# # Create self reference ingress rule to allow +# # communication among resources in the security group. +# ingress { +# from_port = 0 +# to_port = 0 +# protocol = "all" +# self = true +# } + +# # Dynamic Block to create security group rule from var.sg_ingress +# dynamic "ingress" { +# for_each = var.security_group_rules_ingress + +# content { +# cidr_blocks = ingress.value.cidr +# from_port = ingress.value.from_port +# to_port = ingress.value.to_port +# protocol = ingress.value.protocol +# } + +# } + +# # Terraform removes the default ALLOW ALL egress. Let's recreate this +# egress { +# cidr_blocks = ["0.0.0.0/0"] +# from_port = 0 +# to_port = 0 +# protocol = "all" +# } +# } + +# # Knox SG +# resource "aws_security_group" "cdp_knox_sg" { +# vpc_id = aws_vpc.pvc_base.id +# name = var.security_group_knox_name +# description = var.security_group_knox_name + +# tags = merge(var.env_tags, { Name = var.security_group_knox_name }) + +# # Create self reference ingress rule to allow +# # communication among resources in the security group. +# ingress { +# from_port = 0 +# to_port = 0 +# protocol = "all" +# self = true +# } + +# # Dynamic Block to create security group rule from var.sg_ingress +# dynamic "ingress" { +# for_each = var.security_group_rules_ingress + +# content { +# cidr_blocks = ingress.value.cidr +# from_port = ingress.value.from_port +# to_port = ingress.value.to_port +# protocol = ingress.value.protocol +# } + +# } + +# # Terraform removes the default ALLOW ALL egress. Let's recreate this +# egress { +# cidr_blocks = ["0.0.0.0/0"] +# from_port = 0 +# to_port = 0 +# protocol = "all" +# } +# } diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf new file mode 100644 index 0000000..b1186dc --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf @@ -0,0 +1,153 @@ +# ------- General and Provider Resources ------- + +variable "asset_tags" { + type = map(any) + default = {} + description = "Map of tags applied to all cloud-provider assets" +} + +variable "ssh_public_key_file" { + type = string + description = "Local SSH public key file" +} + +variable "prefix" { + type = string + description = "Deployment prefix for all cloud-provider assets" + + validation { + condition = length(var.prefix) < 8 || length(var.prefix) > 4 + error_message = "Valid length for prefix is between 4-7 characters." + } +} + +variable "region" { + type = string + description = "Target AWS region" +} + +# ------- Network Resources ------- +# VPC infrastructure +variable "vpc_name" { + type = string + description = "VPC name" + default = "" +} + +# TODO Convert to list of CIDR blocks +variable "vpc_cidr" { + type = string + description = "VPC CIDR Block" + default = "10.10.0.0/16" +} + +variable "igw_name" { + type = string + description = "Internet Gateway name" + default = "" +} + +# Public Network infrastructure +variable "public_subnet_name" { + type = string + description = "Public Subnet name prefix" + default = "" +} + +variable "public_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Public Subnet details (name, CIDR, AZ, add'l tags)" + default = [] +} + +variable "public_route_table_name" { + type = string + description = "Public Route Table name prefix" + default = "" +} + +# Private Network infrastructure +variable "private_subnet_name" { + type = string + description = "Private Subnet name prefix" + default = "" +} + +variable "private_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Private Subnet details (name, CIDR, AZ, add'l tags)" + default = [] +} + +variable "nat_gateway_name" { + type = string + + description = "NAT gateway name" + default = "" +} + +variable "private_route_table_name" { + type = string + + description = "Private Route Table name prefix" + default = "" +} + +# # Security Groups +# variable "security_group_default_name" { +# type = string + +# description = "Default Security Group for CDP environment" +# } + +# variable "security_group_knox_name" { +# type = string + +# description = "Knox Security Group for CDP environment" +# } + +# variable "security_group_rules_ingress" { +# type = list(object({ +# cidr = list(string) +# from_port = string +# to_port = string +# protocol = string +# })) + +# description = "Ingress rule details for Security Group" +# default = [] +# } + +# # ------- Storage Resources ------- +# variable "storage_locations" { +# type = list(object({ +# bucket = string +# object = string +# })) + +# description = "Storage locations for CDP environment" +# } + +# variable "teardown_deletes_data" { +# type = bool + +# description = "Purge storage locations during teardown" +# } + +# variable "utility_bucket" { +# type = string + +# description = "Utility bucket used as a mirror for downloaded PvC parcels" +# } From b9afbfa9a98ac9e2b5fb576b212495b9e8d34c50 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:00 -0500 Subject: [PATCH 10/45] Update .gitignore for Terraform artifacts Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/.gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/pvc-base-on-aws/.gitignore index e5a1363..7fda933 100644 --- a/private-cloud/pvc-base-on-aws/.gitignore +++ b/private-cloud/pvc-base-on-aws/.gitignore @@ -13,5 +13,11 @@ tf_deployment* *.tfstate *.tfstate.* +# .lock files +*.terraform.lock.hcl + +# .tfvars files +*.tfvars + # Static inventory files -inventory_static* \ No newline at end of file +inventory_static* From 8f542b006b4554020828fa2feaa6fa4758bc84bd Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:01 -0500 Subject: [PATCH 11/45] Remove CDP credentials and add and mount SSH private key file in the Ansible Navigator configuration Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/ansible-navigator.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml index a83766b..84cf454 100644 --- a/private-cloud/pvc-base-on-aws/ansible-navigator.yml +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -18,8 +18,8 @@ ansible-navigator: environment-variables: pass: - AWS_PROFILE - - CDP_PROFILE - SSH_PUBLIC_KEY_FILE + - SSH_PRIVATE_KEY_FILE - CDP_LICENSE_FILE - IPA_USER - IPA_PASSWORD @@ -37,6 +37,8 @@ ansible-navigator: volume-mounts: - src: "${SSH_PUBLIC_KEY_FILE}" dest: "${SSH_PUBLIC_KEY_FILE}" + - src: "${SSH_PRIVATE_KEY_FILE}" + dest: "${SSH_PRIVATE_KEY_FILE}" - src: "${CDP_LICENSE_FILE}" dest: "${CDP_LICENSE_FILE}" - src: "~/.aws" From 5e08f58773032d5c6f4b4add12cc9caf61ceb456 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:01 -0500 Subject: [PATCH 12/45] Remove config.yml and add config-template.yml Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/.gitignore | 3 +++ .../pvc-base-on-aws/{config.yml => config-template.yml} | 0 2 files changed, 3 insertions(+) rename private-cloud/pvc-base-on-aws/{config.yml => config-template.yml} (100%) diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/pvc-base-on-aws/.gitignore index 7fda933..67263b1 100644 --- a/private-cloud/pvc-base-on-aws/.gitignore +++ b/private-cloud/pvc-base-on-aws/.gitignore @@ -21,3 +21,6 @@ tf_deployment* # Static inventory files inventory_static* + +# Per-user configuration files +config.yml diff --git a/private-cloud/pvc-base-on-aws/config.yml b/private-cloud/pvc-base-on-aws/config-template.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/config.yml rename to private-cloud/pvc-base-on-aws/config-template.yml From a06ade1f6a58ec8eea8ace7b5784bfbb70d3f300 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:01 -0500 Subject: [PATCH 13/45] Update to include basic security group for intra-cluster Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_cluster/network.tf | 123 +++++++----------- .../pvc-base-on-aws/tf_cluster/variables.tf | 21 ++- 2 files changed, 53 insertions(+), 91 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf index b2f1011..05abe9a 100644 --- a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf +++ b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf @@ -38,9 +38,12 @@ locals { az = data.aws_availability_zones.pvc_base.names[0], tags = {} }]) + sg_intra_name = var.security_group_intra_name != "" ? var.security_group_intra_name : "${var.prefix}-pvc-base-intra" } # ------- AWS VPC ------- + +# Virtual Cluster resource "aws_vpc" "pvc_base" { cidr_block = var.vpc_cidr tags = { Name = local.vpc_name } @@ -93,6 +96,7 @@ resource "aws_route_table_association" "pvc_base_public" { } # ------- AWS Private Networking infrastructure ------- + # Network Gateways (NAT) resource "aws_eip" "pvc_base" { count = length(aws_subnet.pvc_base_public) @@ -142,81 +146,44 @@ resource "aws_route_table_association" "pvc_base_private" { route_table_id = aws_route_table.pvc_base_private[each.key].id } -# # ------- Security Groups ------- -# # Default SG -# resource "aws_security_group" "cdp_default_sg" { -# vpc_id = aws_vpc.pvc_base.id -# name = var.security_group_default_name -# description = var.security_group_default_name - -# tags = merge(var.env_tags, { Name = var.security_group_default_name }) - -# # Create self reference ingress rule to allow -# # communication among resources in the security group. -# ingress { -# from_port = 0 -# to_port = 0 -# protocol = "all" -# self = true -# } - -# # Dynamic Block to create security group rule from var.sg_ingress -# dynamic "ingress" { -# for_each = var.security_group_rules_ingress - -# content { -# cidr_blocks = ingress.value.cidr -# from_port = ingress.value.from_port -# to_port = ingress.value.to_port -# protocol = ingress.value.protocol -# } - -# } - -# # Terraform removes the default ALLOW ALL egress. Let's recreate this -# egress { -# cidr_blocks = ["0.0.0.0/0"] -# from_port = 0 -# to_port = 0 -# protocol = "all" -# } -# } - -# # Knox SG -# resource "aws_security_group" "cdp_knox_sg" { -# vpc_id = aws_vpc.pvc_base.id -# name = var.security_group_knox_name -# description = var.security_group_knox_name - -# tags = merge(var.env_tags, { Name = var.security_group_knox_name }) - -# # Create self reference ingress rule to allow -# # communication among resources in the security group. -# ingress { -# from_port = 0 -# to_port = 0 -# protocol = "all" -# self = true -# } - -# # Dynamic Block to create security group rule from var.sg_ingress -# dynamic "ingress" { -# for_each = var.security_group_rules_ingress - -# content { -# cidr_blocks = ingress.value.cidr -# from_port = ingress.value.from_port -# to_port = ingress.value.to_port -# protocol = ingress.value.protocol -# } - -# } - -# # Terraform removes the default ALLOW ALL egress. Let's recreate this -# egress { -# cidr_blocks = ["0.0.0.0/0"] -# from_port = 0 -# to_port = 0 -# protocol = "all" -# } -# } +# ------- Security Groups ------- + +# Intra-cluster traffic +resource "aws_security_group" "pvc_base" { + vpc_id = aws_vpc.pvc_base.id + name = local.sg_intra_name + description = "Intra-cluster communication for PVC cluster '${var.prefix}'" + + tags = { Name = local.sg_intra_name } + + # Create self reference ingress rule to allow + # communication among resources in the security group. + ingress { + from_port = 0 + to_port = 0 + protocol = "all" + self = true + } + + # TODO Ingress should be defined by tf_proxied_cluster or other root module + # # Dynamic Block to create security group rule from var.sg_ingress + # dynamic "ingress" { + # for_each = var.security_group_rules_ingress + + # content { + # cidr_blocks = ingress.value.cidr + # from_port = ingress.value.from_port + # to_port = ingress.value.to_port + # protocol = ingress.value.protocol + # } + + # } + + # Terraform removes the default ALLOW ALL egress. Let's recreate this + egress { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + protocol = "all" + } +} diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf index b1186dc..243da24 100644 --- a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf @@ -105,20 +105,15 @@ variable "private_route_table_name" { default = "" } -# # Security Groups -# variable "security_group_default_name" { -# type = string - -# description = "Default Security Group for CDP environment" -# } - -# variable "security_group_knox_name" { -# type = string +# Security Groups +variable "security_group_intra_name" { + type = string -# description = "Knox Security Group for CDP environment" -# } + description = "Security Group for intra-cluster communication" + default = "" +} -# variable "security_group_rules_ingress" { +# variable "security_group_intra_ingress" { # type = list(object({ # cidr = list(string) # from_port = string @@ -126,7 +121,7 @@ variable "private_route_table_name" { # protocol = string # })) -# description = "Ingress rule details for Security Group" +# description = "Ingress rule details for intra-cluster Security Group" # default = [] # } From d2560e9e515ae726b4effc8fffa3b689394c6056 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:01 -0500 Subject: [PATCH 14/45] Add intra-cluster security group roles for ingress and egress Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_cluster/network.tf | 72 ++++++++----------- .../pvc-base-on-aws/tf_cluster/variables.tf | 48 ++++++------- 2 files changed, 54 insertions(+), 66 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf index 05abe9a..68a6d37 100644 --- a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf +++ b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf @@ -19,23 +19,23 @@ data "aws_availability_zones" "pvc_base" { } locals { - vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" - igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" - nat_name = var.nat_gateway_name != "" ? var.nat_gateway_name : "${var.prefix}-pvc-base-nat" - rt_public_name = var.public_route_table_name != "" ? var.public_route_table_name : "${var.prefix}-pvc-base-public" - rt_private_name = var.private_route_table_name != "" ? var.private_route_table_name : "${var.prefix}-pvc-base-private" + vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" + igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" + nat_name = var.nat_gateway_name != "" ? var.nat_gateway_name : "${var.prefix}-pvc-base-nat" + rt_public_name = var.public_route_table_name != "" ? var.public_route_table_name : "${var.prefix}-pvc-base-public" + rt_private_name = var.private_route_table_name != "" ? var.private_route_table_name : "${var.prefix}-pvc-base-private" public_subnet_name = var.public_subnet_name != "" ? var.public_subnet_name : "${var.prefix}-pvc-base-public" public_subnets = length(var.public_subnets) > 0 ? var.public_subnets : tolist([{ name = "${local.public_subnet_name}-01", cidr = cidrsubnet(var.vpc_cidr, 8, 0), - az = data.aws_availability_zones.pvc_base.names[0], + az = data.aws_availability_zones.pvc_base.names[0], tags = {} }]) private_subnet_name = var.private_subnet_name != "" ? var.private_subnet_name : "${var.prefix}-pvc-base-private" private_subnets = length(var.private_subnets) > 0 ? var.private_subnets : tolist([{ name = "${local.private_subnet_name}-01", cidr = cidrsubnet(var.vpc_cidr, 8, 1), - az = data.aws_availability_zones.pvc_base.names[0], + az = data.aws_availability_zones.pvc_base.names[0], tags = {} }]) sg_intra_name = var.security_group_intra_name != "" ? var.security_group_intra_name : "${var.prefix}-pvc-base-intra" @@ -105,12 +105,12 @@ resource "aws_eip" "pvc_base" { } resource "aws_nat_gateway" "pvc_base" { - for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } + for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } subnet_id = each.value.id allocation_id = aws_eip.pvc_base[each.key].id connectivity_type = "public" - tags = { Name = format("%s-%02d", local.nat_name, each.key + 1) } + tags = { Name = format("%s-%02d", local.nat_name, each.key + 1) } } # Private Subnets @@ -133,7 +133,7 @@ resource "aws_route_table" "pvc_base_private" { tags = { Name = format("%s-%02d", local.rt_private_name, index(local.private_subnets, each.value)) } route { - cidr_block = "0.0.0.0/0" + cidr_block = "0.0.0.0/0" nat_gateway_id = aws_nat_gateway.pvc_base[(index(local.private_subnets, each.value) % length(aws_nat_gateway.pvc_base))].id } } @@ -152,38 +152,26 @@ resource "aws_route_table_association" "pvc_base_private" { resource "aws_security_group" "pvc_base" { vpc_id = aws_vpc.pvc_base.id name = local.sg_intra_name - description = "Intra-cluster communication for PVC cluster '${var.prefix}'" - - tags = { Name = local.sg_intra_name } + description = "Intra-cluster communication [${var.prefix}]" + tags = { Name = local.sg_intra_name } +} - # Create self reference ingress rule to allow - # communication among resources in the security group. - ingress { - from_port = 0 - to_port = 0 - protocol = "all" - self = true - } +resource "aws_security_group_rule" "pvc_base_self_ingress" { + security_group_id = aws_security_group.pvc_base.id + type = "ingress" + from_port = 0 + to_port = 0 + description = "Self-reference ingress rule" + protocol = "all" + self = true +} - # TODO Ingress should be defined by tf_proxied_cluster or other root module - # # Dynamic Block to create security group rule from var.sg_ingress - # dynamic "ingress" { - # for_each = var.security_group_rules_ingress - - # content { - # cidr_blocks = ingress.value.cidr - # from_port = ingress.value.from_port - # to_port = ingress.value.to_port - # protocol = ingress.value.protocol - # } - - # } - - # Terraform removes the default ALLOW ALL egress. Let's recreate this - egress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - to_port = 0 - protocol = "all" - } +resource "aws_security_group_rule" "pvc_base_self_egress" { + security_group_id = aws_security_group.pvc_base.id + type = "egress" + from_port = 0 + to_port = 0 + description = "Self-reference egress rule" + protocol = "all" + self = true } diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf index 243da24..cddf569 100644 --- a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf @@ -31,20 +31,20 @@ variable "region" { variable "vpc_name" { type = string description = "VPC name" - default = "" + default = "" } # TODO Convert to list of CIDR blocks variable "vpc_cidr" { type = string description = "VPC CIDR Block" - default = "10.10.0.0/16" + default = "10.10.0.0/16" } variable "igw_name" { type = string description = "Internet Gateway name" - default = "" + default = "" } # Public Network infrastructure @@ -55,15 +55,15 @@ variable "public_subnet_name" { } variable "public_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) })) - + description = "List of Public Subnet details (name, CIDR, AZ, add'l tags)" - default = [] + default = [] } variable "public_route_table_name" { @@ -80,37 +80,37 @@ variable "private_subnet_name" { } variable "private_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) })) description = "List of Private Subnet details (name, CIDR, AZ, add'l tags)" - default = [] + default = [] } variable "nat_gateway_name" { - type = string - + type = string + description = "NAT gateway name" default = "" } variable "private_route_table_name" { - type = string - + type = string + description = "Private Route Table name prefix" default = "" } # Security Groups variable "security_group_intra_name" { - type = string - + type = string + description = "Security Group for intra-cluster communication" - default = "" + default = "" } # variable "security_group_intra_ingress" { @@ -120,7 +120,7 @@ variable "security_group_intra_name" { # to_port = string # protocol = string # })) - + # description = "Ingress rule details for intra-cluster Security Group" # default = [] # } From 5d980bc2b10a531b0cf2cab2f8e51d9fa20f793c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:01 -0500 Subject: [PATCH 15/45] Refactor tf_cluster to tf_hosts, limiting scope to only instances Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_cluster/main.tf | 97 ---------- .../pvc-base-on-aws/tf_cluster/network.tf | 177 ------------------ .../pvc-base-on-aws/tf_cluster/variables.tf | 148 --------------- .../pvc-base-on-aws/tf_hosts/main.tf | 49 +++++ .../pvc-base-on-aws/tf_hosts/outputs.tf | 14 ++ .../pvc-base-on-aws/tf_hosts/variables.tf | 92 +++++++++ 6 files changed, 155 insertions(+), 422 deletions(-) delete mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/main.tf delete mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/network.tf delete mode 100644 private-cloud/pvc-base-on-aws/tf_cluster/variables.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_hosts/main.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_hosts/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_cluster/main.tf deleted file mode 100644 index 68db9c3..0000000 --- a/private-cloud/pvc-base-on-aws/tf_cluster/main.tf +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2023 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 0.13" - required_providers { - aws = { - source = "hashicorp/aws", - version = ">= 4.60.0", - } - ansible = { - source = "ansible/ansible" - version = ">= 1.0.0" - } - } -} - -provider "aws" { - region = var.region - default_tags { - tags = var.asset_tags - } -} - -data "local_file" "ssh_public_key_file" { - filename = var.ssh_public_key_file -} - -resource "aws_key_pair" "deployment" { - key_name = "${var.prefix}-pvc-base" - public_key = data.local_file.ssh_public_key_file.content -} - -# data "aws_ami" "ubuntu" { -# most_recent = true - -# filter { -# name = "name" -# values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] -# } - -# filter { -# name = "virtualization-type" -# values = ["hvm"] -# } - -# owners = ["099720109477"] # Canonical -# } - -# resource "aws_security_group" "bastion" { -# name = "${var.prefix}-private-cloud-base-bastion" -# description = "Allow SSH traffic" -# vpc_id = var.vpc_id -# } - -# resource "aws_vpc_security_group_egress_rule" "cluster" { -# security_group_id = aws_security_group.bastion.id - -# cidr_ipv4 = "0.0.0.0/0" -# ip_protocol = -1 -# description = "All traffic" -# } - -# resource "aws_instance" "bastion" { -# ami = data.aws_ami.ubuntu.id -# instance_type = "t2.micro" -# key_name = aws_key_pair.deployment.key_name -# subnet_id = var.subnet_id -# vpc_security_group_ids = [ -# aws_security_group.bastion.id, -# ] - -# associate_public_ip_address = true - -# tags = { -# Name = "${var.prefix}-private-cloud-base-bastion" -# } -# } - -# resource "ansible_host" "bastion" { -# name = aws_instance.bastion.public_dns -# groups = ["bastion"] -# variables = { -# ansible_user = "ubuntu" -# } -# } diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf b/private-cloud/pvc-base-on-aws/tf_cluster/network.tf deleted file mode 100644 index 68a6d37..0000000 --- a/private-cloud/pvc-base-on-aws/tf_cluster/network.tf +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2023 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO Move VPC creation to tf_proxied_cluster root module - -data "aws_availability_zones" "pvc_base" { - state = "available" -} - -locals { - vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" - igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" - nat_name = var.nat_gateway_name != "" ? var.nat_gateway_name : "${var.prefix}-pvc-base-nat" - rt_public_name = var.public_route_table_name != "" ? var.public_route_table_name : "${var.prefix}-pvc-base-public" - rt_private_name = var.private_route_table_name != "" ? var.private_route_table_name : "${var.prefix}-pvc-base-private" - public_subnet_name = var.public_subnet_name != "" ? var.public_subnet_name : "${var.prefix}-pvc-base-public" - public_subnets = length(var.public_subnets) > 0 ? var.public_subnets : tolist([{ - name = "${local.public_subnet_name}-01", - cidr = cidrsubnet(var.vpc_cidr, 8, 0), - az = data.aws_availability_zones.pvc_base.names[0], - tags = {} - }]) - private_subnet_name = var.private_subnet_name != "" ? var.private_subnet_name : "${var.prefix}-pvc-base-private" - private_subnets = length(var.private_subnets) > 0 ? var.private_subnets : tolist([{ - name = "${local.private_subnet_name}-01", - cidr = cidrsubnet(var.vpc_cidr, 8, 1), - az = data.aws_availability_zones.pvc_base.names[0], - tags = {} - }]) - sg_intra_name = var.security_group_intra_name != "" ? var.security_group_intra_name : "${var.prefix}-pvc-base-intra" -} - -# ------- AWS VPC ------- - -# Virtual Cluster -resource "aws_vpc" "pvc_base" { - cidr_block = var.vpc_cidr - tags = { Name = local.vpc_name } - instance_tenancy = "default" - enable_dns_support = true - enable_dns_hostnames = true -} - -# Internet Gateway -resource "aws_internet_gateway" "pvc_base" { - vpc_id = aws_vpc.pvc_base.id - tags = { Name = local.igw_name } -} - -# ------- AWS Public Network infrastructure ------- - -# TODO Calculate a local for a single default public subnet or defer to the root tf_proxied_cluter module -# TODO Lookup AZ id by input (name) and if not found, use AZ value as ID - -# Public Subnets -resource "aws_subnet" "pvc_base_public" { - for_each = { for idx, subnet in local.public_subnets : idx => subnet } - - vpc_id = aws_vpc.pvc_base.id - cidr_block = each.value.cidr - map_public_ip_on_launch = true - availability_zone = each.value.az - tags = merge(each.value.tags, { Name = each.value.name }) -} - -resource "aws_route_table" "pvc_base_public" { - for_each = { for idx, subnet in local.public_subnets : idx => subnet } - - vpc_id = aws_vpc.pvc_base.id - - tags = { Name = format("%s-%02d", local.rt_public_name, index(local.public_subnets, each.value) + 1) } - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.pvc_base.id - } -} - -# Public Route Table Associations -resource "aws_route_table_association" "pvc_base_public" { - for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } - - subnet_id = each.value.id - route_table_id = aws_route_table.pvc_base_public[each.key].id -} - -# ------- AWS Private Networking infrastructure ------- - -# Network Gateways (NAT) -resource "aws_eip" "pvc_base" { - count = length(aws_subnet.pvc_base_public) - - tags = { Name = format("%s-%02d", local.nat_name, count.index + 1) } -} - -resource "aws_nat_gateway" "pvc_base" { - for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } - - subnet_id = each.value.id - allocation_id = aws_eip.pvc_base[each.key].id - connectivity_type = "public" - tags = { Name = format("%s-%02d", local.nat_name, each.key + 1) } -} - -# Private Subnets -resource "aws_subnet" "pvc_base_private" { - for_each = { for idx, subnet in local.private_subnets : idx => subnet } - - vpc_id = aws_vpc.pvc_base.id - cidr_block = each.value.cidr - map_public_ip_on_launch = false - availability_zone = each.value.az - tags = merge(each.value.tags, { Name = each.value.name }) -} - -# Private Route Tables -resource "aws_route_table" "pvc_base_private" { - for_each = { for idx, subnet in local.private_subnets : idx => subnet } - - vpc_id = aws_vpc.pvc_base.id - - tags = { Name = format("%s-%02d", local.rt_private_name, index(local.private_subnets, each.value)) } - - route { - cidr_block = "0.0.0.0/0" - nat_gateway_id = aws_nat_gateway.pvc_base[(index(local.private_subnets, each.value) % length(aws_nat_gateway.pvc_base))].id - } -} - -# Private Route Table Associations -resource "aws_route_table_association" "pvc_base_private" { - for_each = { for idx, subnet in aws_subnet.pvc_base_private : idx => subnet } - - subnet_id = each.value.id - route_table_id = aws_route_table.pvc_base_private[each.key].id -} - -# ------- Security Groups ------- - -# Intra-cluster traffic -resource "aws_security_group" "pvc_base" { - vpc_id = aws_vpc.pvc_base.id - name = local.sg_intra_name - description = "Intra-cluster communication [${var.prefix}]" - tags = { Name = local.sg_intra_name } -} - -resource "aws_security_group_rule" "pvc_base_self_ingress" { - security_group_id = aws_security_group.pvc_base.id - type = "ingress" - from_port = 0 - to_port = 0 - description = "Self-reference ingress rule" - protocol = "all" - self = true -} - -resource "aws_security_group_rule" "pvc_base_self_egress" { - security_group_id = aws_security_group.pvc_base.id - type = "egress" - from_port = 0 - to_port = 0 - description = "Self-reference egress rule" - protocol = "all" - self = true -} diff --git a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf deleted file mode 100644 index cddf569..0000000 --- a/private-cloud/pvc-base-on-aws/tf_cluster/variables.tf +++ /dev/null @@ -1,148 +0,0 @@ -# ------- General and Provider Resources ------- - -variable "asset_tags" { - type = map(any) - default = {} - description = "Map of tags applied to all cloud-provider assets" -} - -variable "ssh_public_key_file" { - type = string - description = "Local SSH public key file" -} - -variable "prefix" { - type = string - description = "Deployment prefix for all cloud-provider assets" - - validation { - condition = length(var.prefix) < 8 || length(var.prefix) > 4 - error_message = "Valid length for prefix is between 4-7 characters." - } -} - -variable "region" { - type = string - description = "Target AWS region" -} - -# ------- Network Resources ------- -# VPC infrastructure -variable "vpc_name" { - type = string - description = "VPC name" - default = "" -} - -# TODO Convert to list of CIDR blocks -variable "vpc_cidr" { - type = string - description = "VPC CIDR Block" - default = "10.10.0.0/16" -} - -variable "igw_name" { - type = string - description = "Internet Gateway name" - default = "" -} - -# Public Network infrastructure -variable "public_subnet_name" { - type = string - description = "Public Subnet name prefix" - default = "" -} - -variable "public_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) - })) - - description = "List of Public Subnet details (name, CIDR, AZ, add'l tags)" - default = [] -} - -variable "public_route_table_name" { - type = string - description = "Public Route Table name prefix" - default = "" -} - -# Private Network infrastructure -variable "private_subnet_name" { - type = string - description = "Private Subnet name prefix" - default = "" -} - -variable "private_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) - })) - - description = "List of Private Subnet details (name, CIDR, AZ, add'l tags)" - default = [] -} - -variable "nat_gateway_name" { - type = string - - description = "NAT gateway name" - default = "" -} - -variable "private_route_table_name" { - type = string - - description = "Private Route Table name prefix" - default = "" -} - -# Security Groups -variable "security_group_intra_name" { - type = string - - description = "Security Group for intra-cluster communication" - default = "" -} - -# variable "security_group_intra_ingress" { -# type = list(object({ -# cidr = list(string) -# from_port = string -# to_port = string -# protocol = string -# })) - -# description = "Ingress rule details for intra-cluster Security Group" -# default = [] -# } - -# # ------- Storage Resources ------- -# variable "storage_locations" { -# type = list(object({ -# bucket = string -# object = string -# })) - -# description = "Storage locations for CDP environment" -# } - -# variable "teardown_deletes_data" { -# type = bool - -# description = "Purge storage locations during teardown" -# } - -# variable "utility_bucket" { -# type = string - -# description = "Utility bucket used as a mirror for downloaded PvC parcels" -# } diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/main.tf b/private-cloud/pvc-base-on-aws/tf_hosts/main.tf new file mode 100644 index 0000000..8007e2f --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_hosts/main.tf @@ -0,0 +1,49 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + instance_name = var.name != "" ? var.name : "${var.prefix}-pvc-base" +} + +data "aws_ami" "pvc_base" { + filter { + name = "image-id" + values = [ var.image_id ] + } +} + +resource "aws_instance" "pvc_base" { + count = var.quantity + + key_name = var.ssh_key_pair + instance_type = var.instance_type + ami = data.aws_ami.pvc_base.id + + subnet_id = var.subnet_ids[count.index % length(var.subnet_ids)] + associate_public_ip_address = var.public_ip + + security_groups = var.security_groups + + root_block_device { + delete_on_termination = var.root_volume.delete_on_termination + volume_size = var.root_volume.volume_size + volume_type = var.root_volume.volume_type + } + + metadata_options { + http_tokens = data.aws_ami.pvc_base.imds_support == "v2.0" ? "required" : "optional" + } + + tags = merge(var.tags, { Name = format("%s-%02d", local.instance_name, count.index + var.offset + 1) }) +} diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf new file mode 100644 index 0000000..63d83bb --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf @@ -0,0 +1,14 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf b/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf new file mode 100644 index 0000000..f4e2b4c --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf @@ -0,0 +1,92 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "tags" { + type = map(any) + default = {} + description = "Map of tags applied to all cloud-provider assets." +} + +variable "ssh_key_pair" { + type = string + description = "SSH key pair name" +} + +variable "prefix" { + type = string + description = "Deployment prefix for all cloud-provider assets." + + validation { + condition = length(var.prefix) < 8 || length(var.prefix) > 4 + error_message = "Valid length for prefix is between 4-7 characters." + } +} + +# ------- Instances ------- + +variable "name" { + type = string + description = "Instance name prefix" + default = "" +} + +variable "quantity" { + type = number + description = "Number of instances" + default = 1 +} + +variable "offset" { + type = number + description = "Number offset for instance name" + default = 0 +} + +variable "subnet_ids" { + type = list(string) + description = "List of subnet IDs to provision the instances" +} + +variable "security_groups" { + type = list(string) + description = "List of security group IDs to attach to the instances" +} + +variable "public_ip" { + type = bool + description = "Flag to assign public IP addresses to the hosts" + default = false +} + +variable "instance_type" { + type = string + description = "Instance type name for the hosts" + default = "t2.micro" +} + +variable "image_id" { + type = string + description = "AMI image ID for the hosts" +} + +variable "root_volume" { + type = object({ + delete_on_termination = optional(bool, true) + volume_size = optional(number, 100) + volume_type = optional(string) + }) + description = "Root volume details" + default = {} +} + From ded72f9e5fff3cb9c010fff66cc2ffb4ea739527 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:02 -0500 Subject: [PATCH 16/45] Update tf_bastion Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_bastion/main.tf | 47 +++++-------------- .../pvc-base-on-aws/tf_bastion/outputs.tf | 28 +++++++---- .../pvc-base-on-aws/tf_bastion/variables.tf | 36 ++++++++++++-- 3 files changed, 61 insertions(+), 50 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf index c810f46..7d3504d 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf @@ -19,30 +19,19 @@ terraform { source = "hashicorp/aws", version = ">= 4.60.0", } - ansible = { - source = "ansible/ansible" - version = ">= 1.0.0" - } } } -provider "aws" { - region = var.region - default_tags { - tags = var.asset_tags - } +locals { + security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" + bastion_instance_name = var.bastion_instance_name != "" ? var.bastion_instance_name : "${var.prefix}-pvc-base-bastion" } -data "local_file" "ssh_public_key_file" { - filename = var.ssh_public_key_file +data "aws_key_pair" "bastion" { + key_name = var.ssh_key_pair } -resource "aws_key_pair" "deployment" { - key_name = "${var.prefix}-private-cloud-base-bastion" - public_key = data.local_file.ssh_public_key_file.content -} - -data "aws_ami" "ubuntu" { +data "aws_ami" "bastion" { most_recent = true filter { @@ -59,7 +48,7 @@ data "aws_ami" "ubuntu" { } resource "aws_security_group" "bastion" { - name = "${var.prefix}-private-cloud-base-bastion" + name = local.security_group_name description = "Allow SSH traffic" vpc_id = var.vpc_id } @@ -83,25 +72,11 @@ resource "aws_vpc_security_group_ingress_rule" "bastion" { } resource "aws_instance" "bastion" { - ami = data.aws_ami.ubuntu.id + ami = data.aws_ami.bastion.id instance_type = "t2.micro" - key_name = aws_key_pair.deployment.key_name + key_name = data.aws_key_pair.bastion.key_name subnet_id = var.subnet_id - vpc_security_group_ids = [ - aws_security_group.bastion.id, - ] - + vpc_security_group_ids = [ aws_security_group.bastion.id ] associate_public_ip_address = true - - tags = { - Name = "${var.prefix}-private-cloud-base-bastion" - } -} - -resource "ansible_host" "bastion" { - name = aws_instance.bastion.public_dns - groups = ["bastion"] - variables = { - ansible_user = "ubuntu" - } + tags = { Name = local.bastion_instance_name } } diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf index 7abb9d8..37315d7 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf @@ -1,16 +1,26 @@ -output "ssh_key_pair" { - value = { - name = aws_key_pair.deployment.key_name - public_key = data.local_file.ssh_public_key_file - fingerprint = aws_key_pair.deployment.fingerprint - } - description = "CDP Private Cloud bastion" -} +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. output "instance" { - value = aws_instance.bastion + value = { + instance = aws_instance.bastion + user = "ubuntu" + } + description = "Bastion Instance" } output "security_group" { value = aws_security_group.bastion + description = "Bastion Security Group" } diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf index e80a249..24dfcbe 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf @@ -1,12 +1,26 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + variable "asset_tags" { type = map(any) default = {} description = "Map of tags applied to all cloud-provider assets." } -variable "ssh_public_key_file" { +variable "ssh_key_pair" { type = string - description = "Local SSH public key file" + description = "SSH key pair name" } variable "prefix" { @@ -21,15 +35,27 @@ variable "prefix" { variable "region" { type = string - description = "Target AWS region" + description = "Region" } variable "vpc_id" { type = string - description = "Target VPC ID" + description = "VPC ID" } variable "subnet_id" { type = string - description = "Target Subnet ID" + description = "Subnet ID" +} + +variable "security_group_name" { + type = string + description = "Name of the bastion security group" + default = "" } + +variable "bastion_instance_name" { + type = string + description = "Name of the bastion instance" + default = "" +} \ No newline at end of file From 2d9127c402be38aa4cf6e4f24b63bfc645c8ce1d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:02 -0500 Subject: [PATCH 17/45] Create tf_network to manage cluster networking only Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_network/main.tf | 171 ++++++++++++++++++ .../pvc-base-on-aws/tf_network/outputs.tf | 33 ++++ .../pvc-base-on-aws/tf_network/variables.tf | 92 ++++++++++ 3 files changed, 296 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/tf_network/main.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_network/outputs.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_network/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_network/main.tf b/private-cloud/pvc-base-on-aws/tf_network/main.tf new file mode 100644 index 0000000..a1cd407 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_network/main.tf @@ -0,0 +1,171 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws", + version = ">= 4.60.0", + } + } +} + +data "aws_availability_zones" "pvc_base" { + state = "available" +} + +data "aws_vpc" "pvc_base" { + id = var.vpc_id +} + +data "aws_internet_gateway" "pvc_base" { + filter { + name = "attachment.vpc-id" + values = [data.aws_vpc.pvc_base.id] + } +} + +locals { + nat_name = var.nat_gateway_name != "" ? var.nat_gateway_name : "${var.prefix}-pvc-base-nat" + rt_public_name = var.public_route_table_name != "" ? var.public_route_table_name : "${var.prefix}-pvc-base-public" + rt_private_name = var.private_route_table_name != "" ? var.private_route_table_name : "${var.prefix}-pvc-base-private" + public_subnet_name = var.public_subnet_name != "" ? var.public_subnet_name : "${var.prefix}-pvc-base-public" + public_subnets = length(var.public_subnets) > 0 ? var.public_subnets : tolist([{ + name = "${local.public_subnet_name}-01", + cidr = cidrsubnet(data.aws_vpc.pvc_base.cidr_block, 8, 0), + az = data.aws_availability_zones.pvc_base.names[0], + tags = {} + }]) + private_subnet_name = var.private_subnet_name != "" ? var.private_subnet_name : "${var.prefix}-pvc-base-private" + private_subnets = length(var.private_subnets) > 0 ? var.private_subnets : tolist([{ + name = "${local.private_subnet_name}-01", + cidr = cidrsubnet(data.aws_vpc.pvc_base.cidr_block, 8, 1), + az = data.aws_availability_zones.pvc_base.names[0], + tags = {} + }]) + sg_intra_name = var.security_group_intra_name != "" ? var.security_group_intra_name : "${var.prefix}-pvc-base-intra" +} + +# ------- AWS Public Network infrastructure ------- + +# TODO Calculate a local for a single default public subnet or defer to the root tf_proxied_cluter module +# TODO Lookup AZ id by input (name) and if not found, use AZ value as ID + +# Public Subnets +resource "aws_subnet" "pvc_base_public" { + for_each = { for idx, subnet in local.public_subnets : idx => subnet } + + vpc_id = data.aws_vpc.pvc_base.id + cidr_block = each.value.cidr + map_public_ip_on_launch = true + availability_zone = each.value.az + tags = merge(each.value.tags, { Name = each.value.name }) +} + +resource "aws_route_table" "pvc_base_public" { + for_each = { for idx, subnet in local.public_subnets : idx => subnet } + + vpc_id = data.aws_vpc.pvc_base.id + + tags = { Name = format("%s-%02d", local.rt_public_name, index(local.public_subnets, each.value) + 1) } + + route { + cidr_block = "0.0.0.0/0" + gateway_id = data.aws_internet_gateway.pvc_base.id + } +} + +# Public Route Table Associations +resource "aws_route_table_association" "pvc_base_public" { + for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } + + subnet_id = each.value.id + route_table_id = aws_route_table.pvc_base_public[each.key].id +} + +# ------- AWS Private Networking infrastructure ------- + +# Network Gateways (NAT) +resource "aws_eip" "pvc_base" { + count = length(aws_subnet.pvc_base_public) + + tags = { Name = format("%s-%02d", local.nat_name, count.index + 1) } +} + +resource "aws_nat_gateway" "pvc_base" { + for_each = { for idx, subnet in aws_subnet.pvc_base_public : idx => subnet } + + subnet_id = each.value.id + allocation_id = aws_eip.pvc_base[each.key].id + connectivity_type = "public" + tags = { Name = format("%s-%02d", local.nat_name, each.key + 1) } +} + +# Private Subnets +resource "aws_subnet" "pvc_base_private" { + for_each = { for idx, subnet in local.private_subnets : idx => subnet } + + vpc_id = data.aws_vpc.pvc_base.id + cidr_block = each.value.cidr + map_public_ip_on_launch = false + availability_zone = each.value.az + tags = merge(each.value.tags, { Name = each.value.name }) +} + +# Private Route Tables +resource "aws_route_table" "pvc_base_private" { + for_each = { for idx, subnet in local.private_subnets : idx => subnet } + + vpc_id = data.aws_vpc.pvc_base.id + + tags = { Name = format("%s-%02d", local.rt_private_name, index(local.private_subnets, each.value) + 1) } + + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.pvc_base[(index(local.private_subnets, each.value) % length(aws_nat_gateway.pvc_base))].id + } +} + +# Private Route Table Associations +resource "aws_route_table_association" "pvc_base_private" { + for_each = { for idx, subnet in aws_subnet.pvc_base_private : idx => subnet } + + subnet_id = each.value.id + route_table_id = aws_route_table.pvc_base_private[each.key].id +} + +# ------- Security Groups ------- + +# Intra-cluster traffic +resource "aws_security_group" "pvc_base" { + vpc_id = data.aws_vpc.pvc_base.id + name = local.sg_intra_name + description = "Intra-cluster communication [${var.prefix}]" + tags = { Name = local.sg_intra_name } +} + +resource "aws_vpc_security_group_ingress_rule" "pvc_base" { + security_group_id = aws_security_group.pvc_base.id + description = "Self-reference ingress rule" + ip_protocol = -1 + referenced_security_group_id = aws_security_group.pvc_base.id +} + +resource "aws_vpc_security_group_egress_rule" "pvc_base" { + security_group_id = aws_security_group.pvc_base.id + description = "Self-reference egress rule" + ip_protocol = -1 + referenced_security_group_id = aws_security_group.pvc_base.id +} diff --git a/private-cloud/pvc-base-on-aws/tf_network/outputs.tf b/private-cloud/pvc-base-on-aws/tf_network/outputs.tf new file mode 100644 index 0000000..4710d7e --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_network/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "availability_zones" { + value = data.aws_availability_zones.pvc_base + description = "AWS Availability Zones" +} + +output "public_subnets" { + value = values(aws_subnet.pvc_base_public) + description = "Cluster public subnets" +} + +output "private_subnets" { + value = values(aws_subnet.pvc_base_private) + description = "Cluster private subnets" +} + +output "intra_cluster_security_group" { + value = aws_security_group.pvc_base + description = "Intra-cluster traffic Security Group" +} diff --git a/private-cloud/pvc-base-on-aws/tf_network/variables.tf b/private-cloud/pvc-base-on-aws/tf_network/variables.tf new file mode 100644 index 0000000..174922a --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_network/variables.tf @@ -0,0 +1,92 @@ +# ------- Required ------- + +variable "asset_tags" { + type = map(any) + default = {} + description = "Map of tags applied to all cloud-provider assets" +} + +variable "prefix" { + type = string + description = "Deployment prefix for all cloud-provider assets" + + validation { + condition = length(var.prefix) < 8 || length(var.prefix) > 4 + error_message = "Valid length for prefix is between 4-7 characters." + } +} + +variable "region" { + type = string + description = "AWS region" +} + +variable "vpc_id" { + type = string + description = "VPC ID" +} + +# ------- Network Resources ------- + +# Public Network infrastructure +variable "public_subnet_name" { + type = string + description = "Public Subnet name prefix" + default = "" +} + +variable "public_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Public Subnet details (name, CIDR, AZ, add'l tags)" + default = [] +} + +variable "public_route_table_name" { + type = string + description = "Public Route Table name prefix" + default = "" +} + +# Private Network infrastructure +variable "private_subnet_name" { + type = string + description = "Private Subnet name prefix" + default = "" +} + +variable "private_subnets" { + type = list(object({ + name = string + cidr = string + az = string + tags = map(string) + })) + + description = "List of Private Subnet details (name, CIDR, AZ, add'l tags)" + default = [] +} + +variable "nat_gateway_name" { + type = string + description = "NAT gateway name" + default = "" +} + +variable "private_route_table_name" { + type = string + description = "Private Route Table name prefix" + default = "" +} + +# Security Groups +variable "security_group_intra_name" { + type = string + description = "Security Group for intra-cluster communication" + default = "" +} From 78c1900e8cba34dddc6011d956f899fb35b9efb8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:02 -0500 Subject: [PATCH 18/45] Create tf_proxied_cluster root module Signed-off-by: Webster Mudge --- .../tf_proxied_cluster/main.tf | 186 ++++++++++++++++++ .../tf_proxied_cluster/outputs.tf | 43 ++++ .../tf_proxied_cluster/variables.tf | 74 +++++++ 3 files changed, 303 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf create mode 100644 private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf new file mode 100644 index 0000000..8070de6 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf @@ -0,0 +1,186 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO Could use prefix groups to add GlobalVPN restrictions on a non-proxied cluster + +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws", + version = ">= 4.60.0", + } + ansible = { + source = "ansible/ansible" + version = ">= 1.0.0" + } + } +} + +provider "aws" { + region = var.region + default_tags { + tags = var.asset_tags + } +} + +locals { + vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" + igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" +} + +# ------- SSH ------- + +data "local_file" "ssh_public_key_file" { + filename = var.ssh_public_key_file +} + +resource "aws_key_pair" "pvc_base" { + key_name = "${var.prefix}-pvc-base" + public_key = data.local_file.ssh_public_key_file.content +} + +# ------- VPC ------- + +resource "aws_vpc" "pvc_base" { + cidr_block = var.vpc_cidr + tags = { Name = local.vpc_name } + instance_tenancy = "default" + enable_dns_support = true + enable_dns_hostnames = true +} + +resource "aws_internet_gateway" "pvc_base" { + vpc_id = aws_vpc.pvc_base.id + tags = { Name = local.igw_name } +} + +# ------- Network ------- + +module "cluster_network" { + source = "../tf_network" + + region = var.region + prefix = var.prefix + vpc_id = aws_vpc.pvc_base.id +} + +resource "aws_vpc_security_group_egress_rule" "pvc_base" { + security_group_id = module.cluster_network.intra_cluster_security_group.id + description = "All traffic" + ip_protocol = -1 + cidr_ipv4 = "0.0.0.0/0" + tags = { Name = "${var.prefix}-pvc-base-intra" } +} + +# ------- Bastion ------- + +module "bastion" { + source = "../tf_bastion" + depends_on = [aws_key_pair.pvc_base, module.cluster_network] + + region = var.region + prefix = var.prefix + vpc_id = aws_vpc.pvc_base.id + subnet_id = module.cluster_network.public_subnets[0].id + ssh_key_pair = aws_key_pair.pvc_base.key_name +} + +resource "aws_vpc_security_group_ingress_rule" "bastion" { + security_group_id = module.cluster_network.intra_cluster_security_group.id + description = "All Bastion traffic" + ip_protocol = -1 + referenced_security_group_id = module.bastion.security_group.id + tags = { Name = "${var.prefix}-pvc-base-bastion" } +} + +resource "aws_vpc_security_group_egress_rule" "bastion" { + security_group_id = module.cluster_network.intra_cluster_security_group.id + description = "All Bastion traffic" + ip_protocol = -1 + referenced_security_group_id = module.bastion.security_group.id + tags = { Name = "${var.prefix}-pvc-base-bastion" } +} + +resource "aws_vpc_security_group_ingress_rule" "cluster" { + security_group_id = module.bastion.security_group.id + description = "All PVC Base traffic" + ip_protocol = -1 + referenced_security_group_id = module.cluster_network.intra_cluster_security_group.id + tags = { Name = "${var.prefix}-pvc-base-cluster" } +} + +# TODO Review if redundant with existing egress +resource "aws_vpc_security_group_egress_rule" "cluster" { + security_group_id = module.bastion.security_group.id + description = "All PVC Base traffic" + ip_protocol = -1 + referenced_security_group_id = module.cluster_network.intra_cluster_security_group.id +} + +# ------- Cluster ------- + +data "aws_ami" "pvc_base" { + owners = ["099720109477"] # Canonical + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +module "masters" { + source = "../tf_hosts" + depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] + + prefix = "${var.prefix}-proxied-cluster-master" + image_id = data.aws_ami.pvc_base.image_id + ssh_key_pair = aws_key_pair.pvc_base.key_name + subnet_ids = module.cluster_network.private_subnets[*].id + security_groups = [module.cluster_network.intra_cluster_security_group.id] + public_ip = false +} + +module "workers" { + source = "../tf_hosts" + depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] + + prefix = "${var.prefix}-proxied-cluster-worker" + quantity = 4 + image_id = data.aws_ami.pvc_base.image_id + ssh_key_pair = aws_key_pair.pvc_base.key_name + subnet_ids = module.cluster_network.private_subnets[*].id + security_groups = [module.cluster_network.intra_cluster_security_group.id] + public_ip = false +} + +module "moar_workers" { + source = "../tf_hosts" + depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] + + prefix = "${var.prefix}-proxied-cluster-worker" + quantity = 4 + offset = 4 + image_id = data.aws_ami.pvc_base.image_id + ssh_key_pair = aws_key_pair.pvc_base.key_name + subnet_ids = module.cluster_network.private_subnets[*].id + security_groups = [module.cluster_network.intra_cluster_security_group.id] + public_ip = false +} diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf new file mode 100644 index 0000000..d686754 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf @@ -0,0 +1,43 @@ +output "ssh_key_pair" { + value = { + name = aws_key_pair.pvc_base.key_name + public_key = data.local_file.ssh_public_key_file + fingerprint = aws_key_pair.pvc_base.fingerprint + } + description = "SSH key" +} + +# output "bastion" { +# value = { +# instance = aws_instance.bastion +# security_group = aws_security_group.bastion +# ssh_key_pair = { +# name = aws_key_pair.pvc_base.key_name +# public_key = data.local_file.ssh_public_key_file +# fingerprint = aws_key_pair.pvc_base.fingerprint +# } +# } +# description = "Bastion host" +# } + +output "availability_zones" { + value = module.cluster_network.availability_zones + description = "AWS Availability Zones" +} + +output "cluster" { + value = { + public_subnets = module.cluster_network.public_subnets + private_subnets = module.cluster_network.private_subnets + intra_cluster_security_group = module.cluster_network.intra_cluster_security_group + } + description = "Private Cloud cluster" +} + +output "bastion" { + value = { + instance = module.bastion.instance + security_group = module.bastion.security_group + } + description = "Bastion host" +} diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf new file mode 100644 index 0000000..908ffe7 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf @@ -0,0 +1,74 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# variable "security_group_intra_ingress" { +# type = list(object({ +# cidr = list(string) +# from_port = string +# to_port = string +# protocol = string +# })) + +# description = "Ingress rule details for intra-cluster Security Group" +# default = [] +# } + +# ------- General and Provider Resources ------- + +variable "asset_tags" { + type = map(any) + default = {} + description = "Map of tags applied to all cloud-provider assets" +} + +variable "ssh_public_key_file" { + type = string + description = "Local SSH public key file" +} + +variable "prefix" { + type = string + description = "Deployment prefix for all cloud-provider assets" + + validation { + condition = length(var.prefix) < 8 || length(var.prefix) > 4 + error_message = "Valid length for prefix is between 4-7 characters." + } +} + +variable "region" { + type = string + description = "AWS region" +} + +# ------- Network Resources ------- + +variable "vpc_name" { + type = string + description = "VPC name" + default = "" +} + +# TODO Convert to list of CIDR blocks +variable "vpc_cidr" { + type = string + description = "VPC CIDR Block" + default = "10.10.0.0/16" +} + +variable "igw_name" { + type = string + description = "Internet Gateway name" + default = "" +} From f5fc8bf3f64dbacd89dfebd098ab5ac830109da8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:02 -0500 Subject: [PATCH 19/45] Update tf_bastion output to remove nested values Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/tf_bastion/main.tf | 14 +++++++------- .../pvc-base-on-aws/tf_bastion/outputs.tf | 14 ++++++++------ .../pvc-base-on-aws/tf_bastion/variables.tf | 8 ++++---- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf index 7d3504d..7b91f8a 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf @@ -23,7 +23,7 @@ terraform { } locals { - security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" + security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" bastion_instance_name = var.bastion_instance_name != "" ? var.bastion_instance_name : "${var.prefix}-pvc-base-bastion" } @@ -72,11 +72,11 @@ resource "aws_vpc_security_group_ingress_rule" "bastion" { } resource "aws_instance" "bastion" { - ami = data.aws_ami.bastion.id - instance_type = "t2.micro" - key_name = data.aws_key_pair.bastion.key_name - subnet_id = var.subnet_id - vpc_security_group_ids = [ aws_security_group.bastion.id ] + ami = data.aws_ami.bastion.id + instance_type = "t2.micro" + key_name = data.aws_key_pair.bastion.key_name + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.bastion.id] associate_public_ip_address = true - tags = { Name = local.bastion_instance_name } + tags = { Name = local.bastion_instance_name } } diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf index 37315d7..c16c723 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -output "instance" { - value = { - instance = aws_instance.bastion - user = "ubuntu" - } +output "host" { + value = aws_instance.bastion description = "Bastion Instance" } +output "user" { + value = "ubuntu" + description = "Bastion user" +} + output "security_group" { - value = aws_security_group.bastion + value = aws_security_group.bastion description = "Bastion Security Group" } diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf index 24dfcbe..a3046dd 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf @@ -49,13 +49,13 @@ variable "subnet_id" { } variable "security_group_name" { - type = string + type = string description = "Name of the bastion security group" - default = "" + default = "" } variable "bastion_instance_name" { - type = string + type = string description = "Name of the bastion instance" - default = "" + default = "" } \ No newline at end of file From 7155baded5c6d91095c159008bbd8e04b82c9a03 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:02 -0500 Subject: [PATCH 20/45] Add missing terraform stanza and output variable for hosts for tf_hosts Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/tf_hosts/main.tf | 12 +++++++++++- private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf | 4 ++++ private-cloud/pvc-base-on-aws/tf_hosts/variables.tf | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/main.tf b/private-cloud/pvc-base-on-aws/tf_hosts/main.tf index 8007e2f..e7333b3 100644 --- a/private-cloud/pvc-base-on-aws/tf_hosts/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_hosts/main.tf @@ -12,6 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws", + version = ">= 4.60.0", + } + } +} + locals { instance_name = var.name != "" ? var.name : "${var.prefix}-pvc-base" } @@ -19,7 +29,7 @@ locals { data "aws_ami" "pvc_base" { filter { name = "image-id" - values = [ var.image_id ] + values = [var.image_id] } } diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf index 63d83bb..2d8127d 100644 --- a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf @@ -12,3 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +output "hosts" { + value = aws_instance.pvc_base + description = "Hosts" +} diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf b/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf index f4e2b4c..24d3813 100644 --- a/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf @@ -59,7 +59,7 @@ variable "subnet_ids" { } variable "security_groups" { - type = list(string) + type = list(string) description = "List of security group IDs to attach to the instances" } From ca9304c101c8833b23e5d32de82a9452b4aaeabf Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:03 -0500 Subject: [PATCH 21/45] Update tf_proxied_cluster for Ansible inventory Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_network/main.tf | 2 +- .../tf_proxied_cluster/main.tf | 151 +++++++++++++++++- .../tf_proxied_cluster/outputs.tf | 2 +- 3 files changed, 146 insertions(+), 9 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_network/main.tf b/private-cloud/pvc-base-on-aws/tf_network/main.tf index a1cd407..f6c0d1e 100644 --- a/private-cloud/pvc-base-on-aws/tf_network/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_network/main.tf @@ -131,7 +131,7 @@ resource "aws_route_table" "pvc_base_private" { vpc_id = data.aws_vpc.pvc_base.id tags = { Name = format("%s-%02d", local.rt_private_name, index(local.private_subnets, each.value) + 1) } - + route { cidr_block = "0.0.0.0/0" nat_gateway_id = aws_nat_gateway.pvc_base[(index(local.private_subnets, each.value) % length(aws_nat_gateway.pvc_base))].id diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf index 8070de6..02c88a7 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf @@ -131,6 +131,7 @@ resource "aws_vpc_security_group_egress_rule" "cluster" { # ------- Cluster ------- +# user = "ubuntu" data "aws_ami" "pvc_base" { owners = ["099720109477"] # Canonical most_recent = true @@ -150,37 +151,173 @@ module "masters" { source = "../tf_hosts" depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] - prefix = "${var.prefix}-proxied-cluster-master" + prefix = var.prefix + name = "${var.prefix}-pvc-base-master" image_id = data.aws_ami.pvc_base.image_id + instance_type = "m5.4xlarge" ssh_key_pair = aws_key_pair.pvc_base.key_name subnet_ids = module.cluster_network.private_subnets[*].id security_groups = [module.cluster_network.intra_cluster_security_group.id] public_ip = false + + root_volume = { + volume_size = 250 + } } module "workers" { source = "../tf_hosts" depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] - prefix = "${var.prefix}-proxied-cluster-worker" - quantity = 4 + prefix = var.prefix + name = "${var.prefix}-pvc-base-worker" + quantity = 2 image_id = data.aws_ami.pvc_base.image_id + instance_type = "c5.2xlarge" ssh_key_pair = aws_key_pair.pvc_base.key_name subnet_ids = module.cluster_network.private_subnets[*].id security_groups = [module.cluster_network.intra_cluster_security_group.id] public_ip = false + + root_volume = { + volume_size = 250 + } } -module "moar_workers" { +module "freeipa" { source = "../tf_hosts" depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] - prefix = "${var.prefix}-proxied-cluster-worker" - quantity = 4 - offset = 4 + prefix = var.prefix + name = "${var.prefix}-pvc-base-freeipa" image_id = data.aws_ami.pvc_base.image_id + instance_type = "m5.large" # TODO Look up via region ssh_key_pair = aws_key_pair.pvc_base.key_name subnet_ids = module.cluster_network.private_subnets[*].id security_groups = [module.cluster_network.intra_cluster_security_group.id] public_ip = false } + +# ------- Ansible Inventory ------- + +resource "ansible_group" "bastion" { + name = "jump_host" +} + +resource "ansible_group" "freeipa" { + name = "freeipa" +} + +resource "ansible_group" "db" { + name = "db_server" +} + +resource "ansible_group" "cm" { + name = "cloudera_manager" +} + +resource "ansible_group" "workers" { + name = "cluster_workers" + variables = { + host_template = "Workers" + } +} + +resource "ansible_group" "masters" { + name = "cluster_masters" + variables = { + host_template = "Masters" + } +} + +resource "ansible_group" "cluster" { + name = "cluster" + children = [ + ansible_group.masters.name, + ansible_group.workers.name + ] + variables = { + tls = "True" + } +} + +resource "ansible_group" "deployment" { + name = "deployment" + children = [ + ansible_group.cluster.name, + ansible_group.cm.name, + ansible_group.db.name, + ansible_group.freeipa.name + ] + variables = { + ansible_ssh_common_args = "-o ProxyCommand='ssh -o User=${module.bastion.user} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q ${module.bastion.host.public_ip}'" + } +} + +resource "ansible_group" "nodes" { + name = "nodes" + children = [ + ansible_group.deployment.name, + ansible_group.bastion.name + ] +} + +resource "ansible_host" "masters" { + for_each = { for idx, host in module.masters.hosts : idx => host } + + name = each.value.tags["Name"] + + groups = [ + ansible_group.masters.name, + ansible_group.cm.name, + ansible_group.db.name + ] + + variables = { + ansible_host = each.value.private_ip + ansible_user = "ubuntu" + } +} + +resource "ansible_host" "workers" { + for_each = { for idx, host in module.workers.hosts : idx => host } + + name = each.value.tags["Name"] + + groups = [ + ansible_group.workers.name + ] + + variables = { + ansible_host = each.value.private_ip + ansible_user = "ubuntu" + } +} + +resource "ansible_host" "freeipa" { + for_each = { for idx, host in module.freeipa.hosts : idx => host } + + name = each.value.tags["Name"] + + groups = [ + ansible_group.freeipa.name + ] + + variables = { + ansible_host = each.value.private_ip + ansible_user = "ubuntu" + } +} + +resource "ansible_host" "bastion" { + name = module.bastion.host.tags["Name"] + + groups = [ + ansible_group.bastion.name + ] + + variables = { + ansible_host = module.bastion.host.public_ip + ansible_user = module.bastion.user + } +} diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf index d686754..fa26ef4 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf @@ -36,7 +36,7 @@ output "cluster" { output "bastion" { value = { - instance = module.bastion.instance + host = module.bastion.host security_group = module.bastion.security_group } description = "Bastion host" From bd95aed0fbfc26ab2050e23d96726182333eff3a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:03 -0500 Subject: [PATCH 22/45] Set ansible_ssh_private_key_file to the SSH_PRIVATE_KEY_FILE environment variable Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/group_vars/all.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/private-cloud/pvc-base-on-aws/group_vars/all.yml b/private-cloud/pvc-base-on-aws/group_vars/all.yml index 19f3184..05f9b7f 100644 --- a/private-cloud/pvc-base-on-aws/group_vars/all.yml +++ b/private-cloud/pvc-base-on-aws/group_vars/all.yml @@ -1,3 +1,20 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- + # If needed, set the following variable to tell Ansible which key to use # ansible_ssh_private_key_file: ~/.ssh/some-private-key + +ansible_ssh_private_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PRIVATE_KEY_FILE') }}" From f8bb2060571ac1651c37283db8cdf2dc5c98205d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:03 -0500 Subject: [PATCH 23/45] Refactor pre_setup_resource.yml playbook into the local role, cluster_reqs Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/pre_setup_resources.yml | 23 -------- .../roles/cluster_reqs/README.md | 38 ++++++++++++++ .../roles/cluster_reqs/defaults/main.yml | 2 + .../roles/cluster_reqs/meta/main.yml | 52 +++++++++++++++++++ .../roles/cluster_reqs/tasks/Ubuntu.yml | 17 ++++++ .../roles/cluster_reqs/tasks/default.yml | 17 ++++++ .../roles/cluster_reqs/tasks/main.yml | 51 ++++++++++++++++++ .../roles/cluster_reqs/tests/inventory | 2 + .../roles/cluster_reqs/tests/test.yml | 5 ++ .../roles/cluster_reqs/vars/RedHat.yml | 24 +++++++++ .../roles/cluster_reqs/vars/Ubuntu.yml | 22 ++++++++ .../roles/cluster_reqs/vars/default.yml | 17 ++++++ .../roles/cluster_reqs/vars/main.yml | 2 + 13 files changed, 249 insertions(+), 23 deletions(-) delete mode 100644 private-cloud/pvc-base-on-aws/pre_setup_resources.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml diff --git a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml deleted file mode 100644 index 91f66b8..0000000 --- a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - # Playbook to setup additional resources for PvC Deployment - -- name: Ensure additional requirements are available - hosts: nodes - gather_facts: no - become: yes - tasks: - - - name: Install required packages - ansible.builtin.package: - name: - - tree - - jq - - zip - - python38 - state: present - - - name: Install required packages - ansible.builtin.pip: - name: psycopg2-binary==2.9.5 - executable: pip3.8 - state: present diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md new file mode 100644 index 0000000..225dd44 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml new file mode 100644 index 0000000..803ff7a --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for cluster_reqs diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml new file mode 100644 index 0000000..c572acc --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml new file mode 100644 index 0000000..66ec3df --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Update apt cache + ansible.builtin.apt: + update_cache: yes diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml new file mode 100644 index 0000000..102e442 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +# no op diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml new file mode 100644 index 0000000..1507c77 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml @@ -0,0 +1,51 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Gather host distribution details + ansible.builtin.setup: + gather_subset: distribution + +- name: Load distribution variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Run distribution tasks + ansible.builtin.include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install required system packages + ansible.builtin.package: + name: "{{ system_packages }}" + state: present + +- name: Install required Python libraries + ansible.builtin.pip: + name: "{{ python_libraries }}" + state: present diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml new file mode 100644 index 0000000..37b6e3c --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - cluster_reqs diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml new file mode 100644 index 0000000..cb60e46 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +system_packages: + - tree + - jq + - zip + - postgresql-devel + - gcc + - platform-python-devel + +python_libraries: + - psycopg2-binary==2.9.5 diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml new file mode 100644 index 0000000..0d0ecd8 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml @@ -0,0 +1,22 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +system_packages: + - tree + - jq + - zip + - python3-pip + +python_libraries: + - psycopg2-binary==2.9.5 \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml new file mode 100644 index 0000000..102e442 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +# no op diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml new file mode 100644 index 0000000..ac1c250 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for cluster_reqs From 1aaa65d53e7532ce54947d1d9ce11b477419691a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:03 -0500 Subject: [PATCH 24/45] Add Terraform variables template for infrastructure Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/templates/infra.tfvars.j2 | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 diff --git a/private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 b/private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 new file mode 100644 index 0000000..d7f682a --- /dev/null +++ b/private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 @@ -0,0 +1,26 @@ +// Copyright 2023 Cloudera, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +ssh_public_key_file = "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" +domain = "{{ domain }}" +prefix = "{{ name_prefix}}" +region = "{{ infra_region }}" + +{% if tags is defined and tags is mapping %} +asset_tags = { +{% for k, v in tags.items() %} + {{ k }} = "{{ v }}" +{% endfor %} +} +{% endif %} From 8f08c4ab495e70151e9ef0532b98c2680f71febf Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:04 -0500 Subject: [PATCH 25/45] Update tf_bastion to accept an outside AMI image id Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_bastion/main.tf | 24 ++++--------------- .../pvc-base-on-aws/tf_bastion/outputs.tf | 9 ++----- .../pvc-base-on-aws/tf_bastion/variables.tf | 14 +++++------ 3 files changed, 13 insertions(+), 34 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf index 7b91f8a..147dfc2 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf @@ -23,30 +23,14 @@ terraform { } locals { - security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" - bastion_instance_name = var.bastion_instance_name != "" ? var.bastion_instance_name : "${var.prefix}-pvc-base-bastion" + security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" + instance_name = var.instance_name != "" ? var.instance_name : "${var.prefix}-pvc-base-bastion" } data "aws_key_pair" "bastion" { key_name = var.ssh_key_pair } -data "aws_ami" "bastion" { - most_recent = true - - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } - - owners = ["099720109477"] # Canonical -} - resource "aws_security_group" "bastion" { name = local.security_group_name description = "Allow SSH traffic" @@ -72,11 +56,11 @@ resource "aws_vpc_security_group_ingress_rule" "bastion" { } resource "aws_instance" "bastion" { - ami = data.aws_ami.bastion.id + ami = var.image_id instance_type = "t2.micro" key_name = data.aws_key_pair.bastion.key_name subnet_id = var.subnet_id vpc_security_group_ids = [aws_security_group.bastion.id] associate_public_ip_address = true - tags = { Name = local.bastion_instance_name } + tags = { Name = local.instance_name } } diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf index c16c723..e20ba3f 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf @@ -13,13 +13,8 @@ # limitations under the License. output "host" { - value = aws_instance.bastion - description = "Bastion Instance" -} - -output "user" { - value = "ubuntu" - description = "Bastion user" + value = aws_instance.bastion + description = "Bastion host" } output "security_group" { diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf index a3046dd..66b5f0b 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf @@ -33,11 +33,6 @@ variable "prefix" { } } -variable "region" { - type = string - description = "Region" -} - variable "vpc_id" { type = string description = "VPC ID" @@ -54,8 +49,13 @@ variable "security_group_name" { default = "" } -variable "bastion_instance_name" { +variable "image_id" { + type = string + description = "AMI image ID for the bastion" +} + +variable "instance_name" { type = string description = "Name of the bastion instance" default = "" -} \ No newline at end of file +} From 8eda884c1c4b666acb71f9a66a0d4b1af9c20cba Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:04 -0500 Subject: [PATCH 26/45] Formatting Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf index 2d8127d..413602d 100644 --- a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf @@ -13,6 +13,6 @@ # limitations under the License. output "hosts" { - value = aws_instance.pvc_base + value = aws_instance.pvc_base description = "Hosts" } From 321b9dca7472810b10f82071942978fc3d9363f5 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:04 -0500 Subject: [PATCH 27/45] License header Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/tf_network/variables.tf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/private-cloud/pvc-base-on-aws/tf_network/variables.tf b/private-cloud/pvc-base-on-aws/tf_network/variables.tf index 174922a..91278f4 100644 --- a/private-cloud/pvc-base-on-aws/tf_network/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_network/variables.tf @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ------- Required ------- variable "asset_tags" { From 4cdb95922f6849b7eeab5562385f204dc44d5ee1 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:04 -0500 Subject: [PATCH 28/45] Update AMI image to RHEL 8.6 Add domain for Ansible hosts Signed-off-by: Webster Mudge --- .../tf_proxied_cluster/main.tf | 74 ++++++++++--------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf index 02c88a7..faa0568 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf @@ -36,10 +36,34 @@ provider "aws" { } locals { + # RedHat 8.6 + ami_user = "ec2-user" + ami_owners = ["309956199498"] + ami_filters = { + name = ["RHEL-8.6*"] + architecture = ["x86_64"] + } + domain = var.domain != "" ? var.domain : "${var.prefix}.pvc-base.cldr.example" vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" } +# ------- AMI ------- + +data "aws_ami" "pvc_base" { + owners = local.ami_owners + most_recent = true + + dynamic "filter" { + for_each = local.ami_filters + + content { + name = filter.key + values = filter.value + } + } +} + # ------- SSH ------- data "local_file" "ssh_public_key_file" { @@ -90,8 +114,8 @@ module "bastion" { source = "../tf_bastion" depends_on = [aws_key_pair.pvc_base, module.cluster_network] - region = var.region prefix = var.prefix + image_id = data.aws_ami.pvc_base.image_id vpc_id = aws_vpc.pvc_base.id subnet_id = module.cluster_network.public_subnets[0].id ssh_key_pair = aws_key_pair.pvc_base.key_name @@ -131,22 +155,6 @@ resource "aws_vpc_security_group_egress_rule" "cluster" { # ------- Cluster ------- -# user = "ubuntu" -data "aws_ami" "pvc_base" { - owners = ["099720109477"] # Canonical - most_recent = true - - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - module "masters" { source = "../tf_hosts" depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] @@ -250,22 +258,22 @@ resource "ansible_group" "deployment" { ansible_group.freeipa.name ] variables = { - ansible_ssh_common_args = "-o ProxyCommand='ssh -o User=${module.bastion.user} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q ${module.bastion.host.public_ip}'" + ansible_ssh_common_args = "-o ProxyCommand='ssh -i {{ lookup('ansible.builtin.env', 'SSH_PRIVATE_KEY_FILE') }} -o User=${local.ami_user} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -q ${module.bastion.host.public_ip}'" } } -resource "ansible_group" "nodes" { - name = "nodes" - children = [ - ansible_group.deployment.name, - ansible_group.bastion.name - ] -} +# resource "ansible_group" "nodes" { +# name = "nodes" +# children = [ +# ansible_group.deployment.name, +# ansible_group.bastion.name +# ] +# } resource "ansible_host" "masters" { for_each = { for idx, host in module.masters.hosts : idx => host } - name = each.value.tags["Name"] + name = format("%s.%s", each.value.tags["Name"], local.domain) groups = [ ansible_group.masters.name, @@ -275,14 +283,14 @@ resource "ansible_host" "masters" { variables = { ansible_host = each.value.private_ip - ansible_user = "ubuntu" + ansible_user = local.ami_user } } resource "ansible_host" "workers" { for_each = { for idx, host in module.workers.hosts : idx => host } - name = each.value.tags["Name"] + name = format("%s.%s", each.value.tags["Name"], local.domain) groups = [ ansible_group.workers.name @@ -290,14 +298,14 @@ resource "ansible_host" "workers" { variables = { ansible_host = each.value.private_ip - ansible_user = "ubuntu" + ansible_user = local.ami_user } } resource "ansible_host" "freeipa" { for_each = { for idx, host in module.freeipa.hosts : idx => host } - name = each.value.tags["Name"] + name = format("%s.%s", each.value.tags["Name"], local.domain) groups = [ ansible_group.freeipa.name @@ -305,12 +313,12 @@ resource "ansible_host" "freeipa" { variables = { ansible_host = each.value.private_ip - ansible_user = "ubuntu" + ansible_user = local.ami_user } } resource "ansible_host" "bastion" { - name = module.bastion.host.tags["Name"] + name = format("%s.%s", module.bastion.host.tags["Name"], local.domain) groups = [ ansible_group.bastion.name @@ -318,6 +326,6 @@ resource "ansible_host" "bastion" { variables = { ansible_host = module.bastion.host.public_ip - ansible_user = module.bastion.user + ansible_user = local.ami_user } } From e6a805915d996748b0ee53726a4a6740240cb86b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:04 -0500 Subject: [PATCH 29/45] Add VPC to output Signed-off-by: Webster Mudge --- .../tf_proxied_cluster/outputs.tf | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf index fa26ef4..3d84753 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + output "ssh_key_pair" { value = { name = aws_key_pair.pvc_base.key_name @@ -7,18 +21,10 @@ output "ssh_key_pair" { description = "SSH key" } -# output "bastion" { -# value = { -# instance = aws_instance.bastion -# security_group = aws_security_group.bastion -# ssh_key_pair = { -# name = aws_key_pair.pvc_base.key_name -# public_key = data.local_file.ssh_public_key_file -# fingerprint = aws_key_pair.pvc_base.fingerprint -# } -# } -# description = "Bastion host" -# } +output "vpc" { + value = aws_vpc.pvc_base + description = "AWS VPC" +} output "availability_zones" { value = module.cluster_network.availability_zones From f2d06f22a97e99bbe70a89a7a357db912123b3cc Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:05 -0500 Subject: [PATCH 30/45] Add domain to variables Signed-off-by: Webster Mudge --- .../tf_proxied_cluster/variables.tf | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf index 908ffe7..baecd9e 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf @@ -12,26 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# variable "security_group_intra_ingress" { -# type = list(object({ -# cidr = list(string) -# from_port = string -# to_port = string -# protocol = string -# })) - -# description = "Ingress rule details for intra-cluster Security Group" -# default = [] -# } - # ------- General and Provider Resources ------- -variable "asset_tags" { - type = map(any) - default = {} - description = "Map of tags applied to all cloud-provider assets" -} - variable "ssh_public_key_file" { type = string description = "Local SSH public key file" @@ -52,6 +34,12 @@ variable "region" { description = "AWS region" } +variable "asset_tags" { + type = map(string) + default = {} + description = "Map of tags applied to all cloud-provider assets" +} + # ------- Network Resources ------- variable "vpc_name" { @@ -72,3 +60,9 @@ variable "igw_name" { description = "Internet Gateway name" default = "" } + +variable "domain" { + type = string + description = "Private subdomain for proxied hosts, e.g. pvc-base.cldr.example" + default = "" +} From 3e47823a8740547c0d4200b7318715e9f7867269 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:05 -0500 Subject: [PATCH 31/45] Add license header Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/.gitignore | 14 +++++++++++ .../pvc-base-on-aws/base_postfix.yml | 14 +++++++++++ private-cloud/pvc-base-on-aws/base_setup.yml | 14 +++++++++++ private-cloud/pvc-base-on-aws/cluster.yml | 14 +++++++++++ .../pvc-base-on-aws/external_setup.yml | 14 +++++++++++ .../pvc-base-on-aws/internal_setup.yml | 14 +++++++++++ .../pvc-base-on-aws/validate_dns_lookups.yml | 24 ++++++++++++++----- 7 files changed, 102 insertions(+), 6 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/pvc-base-on-aws/.gitignore index 67263b1..a925cf1 100644 --- a/private-cloud/pvc-base-on-aws/.gitignore +++ b/private-cloud/pvc-base-on-aws/.gitignore @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Ansible Navigator assets ansible-navigator.log runs diff --git a/private-cloud/pvc-base-on-aws/base_postfix.yml b/private-cloud/pvc-base-on-aws/base_postfix.yml index c7336f5..b047714 100644 --- a/private-cloud/pvc-base-on-aws/base_postfix.yml +++ b/private-cloud/pvc-base-on-aws/base_postfix.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - name: Postfix CDP Private Cloud clusters diff --git a/private-cloud/pvc-base-on-aws/base_setup.yml b/private-cloud/pvc-base-on-aws/base_setup.yml index be6790b..a3bda48 100644 --- a/private-cloud/pvc-base-on-aws/base_setup.yml +++ b/private-cloud/pvc-base-on-aws/base_setup.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - name: Set up CDP Private Cloud clusters diff --git a/private-cloud/pvc-base-on-aws/cluster.yml b/private-cloud/pvc-base-on-aws/cluster.yml index 5e600bd..b3f7a90 100644 --- a/private-cloud/pvc-base-on-aws/cluster.yml +++ b/private-cloud/pvc-base-on-aws/cluster.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- clusters: diff --git a/private-cloud/pvc-base-on-aws/external_setup.yml b/private-cloud/pvc-base-on-aws/external_setup.yml index cdbe151..07c0224 100644 --- a/private-cloud/pvc-base-on-aws/external_setup.yml +++ b/private-cloud/pvc-base-on-aws/external_setup.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - name: Set up CDP Private Cloud external prerequisites diff --git a/private-cloud/pvc-base-on-aws/internal_setup.yml b/private-cloud/pvc-base-on-aws/internal_setup.yml index 18b56ea..45fd301 100644 --- a/private-cloud/pvc-base-on-aws/internal_setup.yml +++ b/private-cloud/pvc-base-on-aws/internal_setup.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - name: Set up CDP Private Cloud internal prerequisites diff --git a/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml b/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml index 236169e..a6ed0a5 100644 --- a/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml +++ b/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml @@ -1,3 +1,19 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + - name: Check that all hosts in inventory are reachable hosts: all gather_facts: no @@ -5,23 +21,19 @@ - name: Ping each host from controller ping: - - name: Check Forward and Reverse DNS lookups hosts: localhost connection: local gather_facts: no tasks: - - - name: Print the list of hosts + - name: Print the hosts debug: var: groups['all'] - - name: Forward lookup + - name: Forward lookup for all hosts ansible.builtin.debug: msg: - "Forward lookup DNS for {{ item }} is {{ lookup('community.general.dig', item , qtype='A') }}" loop: "{{ groups['all'] }}" # TODO Register forward lookup IP and do reverse lookup - - From 2c0c09f6dc1c9fd5781efa0193b3d7b431559d22 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:05 -0500 Subject: [PATCH 32/45] Add Terraform inventory plugin Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/ansible-navigator.yml | 19 +++++++++++++++++++ private-cloud/pvc-base-on-aws/inventory.yml | 16 ++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 private-cloud/pvc-base-on-aws/inventory.yml diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml index 84cf454..98fc070 100644 --- a/private-cloud/pvc-base-on-aws/ansible-navigator.yml +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -1,3 +1,17 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- ansible-navigator: @@ -12,6 +26,11 @@ ansible-navigator: level: debug append: False + ansible: + inventory: + entries: + - inventory.yml + execution-environment: container-engine: docker enabled: True diff --git a/private-cloud/pvc-base-on-aws/inventory.yml b/private-cloud/pvc-base-on-aws/inventory.yml new file mode 100644 index 0000000..d9987a4 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/inventory.yml @@ -0,0 +1,16 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +plugin: cloud.terraform.terraform_provider +project_path: tf_proxied_cluster From ddea76f0036da8c569b62eebe394a1f42bbbaf0e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:05 -0500 Subject: [PATCH 33/45] Remove cloudera.exe.provision assets Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/hostvars.j2 | 9 --------- private-cloud/pvc-base-on-aws/instance_vars.j2 | 13 ------------- 2 files changed, 22 deletions(-) delete mode 100644 private-cloud/pvc-base-on-aws/hostvars.j2 delete mode 100644 private-cloud/pvc-base-on-aws/instance_vars.j2 diff --git a/private-cloud/pvc-base-on-aws/hostvars.j2 b/private-cloud/pvc-base-on-aws/hostvars.j2 deleted file mode 100644 index bf2ca91..0000000 --- a/private-cloud/pvc-base-on-aws/hostvars.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{# Collect and output individual host variables #} -{% macro host_variables(host) %} -{% set fields = [] %} -{% set _ = fields.append("ansible_user=" + host['ansible_user']) if 'ansible_user' in host %} -{% set _ = fields.append("host_template=" + host['host_template']) if 'host_template' in host %} -{% set _ = fields.append("label=" + host['label']) if 'label' in host %} -{% set _ = fields.append("tls=" + host['tls'] | string) if 'tls' in host %} -{{ host['inventory_hostname'] }} {{ fields | join(' ') }} -{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/pvc-base-on-aws/instance_vars.j2 b/private-cloud/pvc-base-on-aws/instance_vars.j2 deleted file mode 100644 index 6e7063f..0000000 --- a/private-cloud/pvc-base-on-aws/instance_vars.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{# Define the metadata tags for the individual Openstack instances #} -{# Output should be TF map _entries_, not a map itself #} - -{% macro instance_tags(host) %} -{% set tags = {} %} -{% set _ = tags.update({ 'ansible_user': host.ansible_user }) if host.ansible_user is defined %} -{% set _ = tags.update({ 'host_template': host.host_template }) if host.host_template is defined %} -{% set _ = tags.update({ 'groups': host.groups | join(', ') }) if host.groups is defined %} -{% set _ = tags.update({ 'tls': host.tls | string }) if host.tls is defined %} -{% for k, v in tags.items() %} - {{ k }} = "{{ v }}"{{ "," if not loop.last else "" }} -{% endfor %} -{%- endmacro %} \ No newline at end of file From 7bccd29492ea055916ddefc2f28bacc7204bac31 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:05 -0500 Subject: [PATCH 34/45] Remove legacy infrastructure configuration and update FreeIPA parameters Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/definition.yml | 153 ++++------ private-cloud/pvc-base-on-aws/pre_setup.yml | 267 ++++++------------ .../pvc-base-on-aws/pre_teardown.yml | 41 +-- 3 files changed, 165 insertions(+), 296 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/pvc-base-on-aws/definition.yml index 8405c6f..9b081de 100644 --- a/private-cloud/pvc-base-on-aws/definition.yml +++ b/private-cloud/pvc-base-on-aws/definition.yml @@ -1,80 +1,88 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- -kerberos_activated: True -encryption_activated: True -# tls_activated: True -# autotls: False +### Infrastructure -repositories: - # Offical CDH 7.1.9.0 - - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/ -cloudera_manager_version: 7.11.3 +infra_type: aws -jdk_version: 11 +# Terraform +terraform: + state: + storage: local + create_remote_storage: False + s3_region: "{{ infra_region }}" + s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" # SSH public_key_id: "{{ owner_prefix }}-{{ name_prefix }}" public_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" public_key_text: "{{ lookup('ansible.builtin.file', lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE')) }}" -# Cloudera license -license_file: "{{ lookup('ansible.builtin.env', 'CDP_LICENSE_FILE') }}" +### Cluster -# Terraform settings -terraform: - state: - storage: local - create_remote_storage: False - s3_region: "{{ infra_region }}" - s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" +kerberos_activated: True +encryption_activated: True +# tls_activated: True +# autotls: False + +jdk_version: 11 -# DNS -dns_provider: "freeipa" +repositories: + - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/ # Offical CDH 7.1.9.0 # Connect FreeIPA to Knox and Ranger freeipa_activated: yes -# FreeIPA client install on cluster nodes is done during pre-setup + +# FreeIPA client install on cluster nodes is done during pre_setup, so suppress freeipa_enroll: no # FreeIPA realm settings freeipa: realm: "{{ realm }}" -##### - -# Vars from legacy cloudera-Deploy -use_auto_repo_mirror: no -use_default_cluster_definition: no -use_download_mirror: no -preload_cm_parcel_repo: yes -teardown_everything: yes - # Defer user and group creation to SSSD/FreeIPA registration -#Use when you dont need ansible to run "ca_server" +# Use when you dont need ansible to run "ca_server" skip_user_group_init: no -# Cloudera Manager details +### Cloudera Manager +cloudera_manager_version: 7.11.3 + +# General options cloudera_manager_options: - CUSTOM_BANNER_HTML: "PvC Base Lab ({{ name_prefix }})" + CUSTOM_BANNER_HTML: "PVC Base Lab ({{ name_prefix }})" SESSION_TIMEOUT: 43200 PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 KRB_AUTH_ENABLE: "true" +# Cloudera license +license_file: "{{ lookup('ansible.builtin.env', 'CDP_LICENSE_FILE') }}" + # License options (this is due to a hardcoded tmp directory on the target/manager node) license_local_tmp_path: /tmp/cloudera_license.txt -############### -### RDBMS ### -############### +### RDBMS + database_type: postgresql database_version: 12 database_tls: true database_default_password: "{{ common_password }}" -############### +### Red Hat FreeIPA -## Red Hat FreeIPA krb5_kdc_type: Red Hat IPA krb5_realm: "{{ realm }}" krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user | default(lookup('env', 'IPA_USER', default='Undefined')) }}@{{ krb5_realm }}" @@ -85,9 +93,7 @@ krb5_enc_types: "aes256-cts aes128-cts" #krb5_kdc_host: "{{ lookup('community.general.dig', '_ldap._tcp.' + domain +'./SRV', 'flat=0', wantlist=True, fail_on_error=True) | map(attribute='target') | first }}" krb5_kdc_host: "{{ groups['freeipa'] | first | default('') }}" -############################################ -### CM External Auth - FreeIPA as LDAP ### -############################################ +### CM External Auth - FreeIPA as LDAP # Settings for FreeIPA sidecar deployment cloudera_manager_external_auth: @@ -134,62 +140,9 @@ ca_server_attrs_root: ca_server_attrs_intermediate: CN: 'SE-Intermediate CA' -# Infrastructure -infra: - aws: - region: "{{ infra_region }}" - enable_dns: no - public_subnets: # TODO: Implement better selection of AZs - - az: "{{ infra_region }}a" - private_subnets: - - az: "{{ infra_region }}b" - - az: "{{ infra_region }}c" - default_security_group: - ingress: - - cidr: [ "0.0.0.0/0" ] - from: 22 - to: 22 - protocol: TCP - desc: SSH (Ansible) - nodes: - # Sidecar FreeIPA for Kerberos and DNS - - hostname_prefix: "freeipa" - count: 1 - subnet_index: 0 # TODO: Implement better selection of AZs. this is the combined list of public + private - groups: - - freeipa - - jump_host - - nodes - instance_type: m5.large - # Manager, Master, DB, and CA - - hostname_prefix: "base" # hostname will be of form {{ name_prefix }}-{{ hostname_prefix }}- - count: 1 - groups: - - cloudera_manager - - cluster_masters - - cluster - - nodes - - db_server - - linux_nodes - - deployment - host_template: Masters - instance_type: m5.4xlarge - root_volume: - volume_size: 250 - subnet_index: 1 - tls: True - # Cluster Worker - - hostname_prefix: worker - count: 2 - groups: - - cluster_workers - - cluster - - nodes - - linux_nodes - - deployment - instance_type: c5.2xlarge - subnet_index: 1 - root_volume: - volume_size: 250 - host_template: Workers - tls: True +# Vars from legacy cloudera-Deploy +use_auto_repo_mirror: no +use_default_cluster_definition: no +use_download_mirror: no +preload_cm_parcel_repo: yes +teardown_everything: yes diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/pvc-base-on-aws/pre_setup.yml index b4035c5..ed322d1 100644 --- a/private-cloud/pvc-base-on-aws/pre_setup.yml +++ b/private-cloud/pvc-base-on-aws/pre_setup.yml @@ -1,231 +1,140 @@ ---- +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# Run variable checker +--- -# Provision the infrastructure at the cloud provider -- name: Provision infrastructure resources +- name: Provision AWS infrastructure hosts: localhost connection: local - gather_facts: yes - module_defaults: - # Set your definition-specific inventory host variables here. - # NOTE: the 'node' variable is in scope and contains each entry coming from - # the Terraform 'nodes' output variable. - ansible.builtin.add_host: - host_template: "{{ node.metadata.host_template | default(omit) }}" - tls: "{{ node.metadata.tls | default(omit) }}" - # TODO Review the SSH args, these are overwritten - ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -C -o ControlMaster=auto -o ControlPersist=1200s -o BatchMode=yes' + gather_facts: no tags: infra tasks: - - name: Provision the infrastructure resources in the cloud provider - when: infra_type != 'static' - ansible.builtin.import_role: - name: cloudera.exe.provision - vars: - provision_provider: "{{ infra_type }}" - provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" - provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" - provision_terraform_parallelism: "{{ terraform.parallelism | default(omit) }}" - provision_state_storage: "{{ terraform.state.storage }}" - provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" - provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" - provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" - provision_name_prefix: "{{ name_prefix }}" - provision_domain_suffix: "{{ domain }}" - provision_ssh_keypair_label: "{{ ssh_keypair.label | default(public_key_id) }}" - provision_ssh_keypair_public_key: "{{ ssh_keypair.public_key | default(public_key_text) }}" - provision_owner_email: "{{ owner_email }}" - provision_openstack_network_name: "{{ infra[infra_type].network_name | default(omit) }}" - provision_openstack_default_image_name: "{{ infra[infra_type].default_image_name | default(omit) }}" - provision_openstack_default_availability_zone: "{{ infra[infra_type].default_availability_zone | default(omit) }}" - provision_aws_ec2_region: "{{ infra[infra_type].region | default(omit) }}" - provision_aws_ec2_vpc_enable_dns_support: "{{ infra[infra_type].enable_dns | default(omit) }}" - provision_aws_ec2_vpc_enable_dns_hostnames: "{{ infra[infra_type].enable_dns | default(omit) }}" - provision_aws_ec2_public_subnets: "{{ infra[infra_type].public_subnets | default(omit) }}" - provision_aws_ec2_private_subnets: "{{ infra[infra_type].private_subnets | default(omit) }}" - provision_aws_ec2_default_security_group_ingress: "{{ infra[infra_type].default_security_group.ingress | default(omit) }}" - provision_instances: "{{ infra[infra_type].nodes | default([]) }}" - provision_tags: "{{ deployment_tags | default(omit) }}" - -# Confirm the availablity of all nodes -- name: Ensure node readiness - hosts: nodes + - name: Set Terraform variables + ansible.builtin.template: + dest: tf_proxied_cluster/terraform.tfvars + src: infra.tfvars.j2 + + - name: Check for existing Terraform state file + ansible.builtin.stat: + path: tf_proxied_cluster/terraform.tfstate + register: __tf_state + + - name: Establish the infrastructure + cloud.terraform.terraform: + project_path: "tf_proxied_cluster/" + state: "present" + force_init: "{{ not __tf_state.stat.exists }}" + provider_upgrade: true + + - name: Establish the Ansible inventory from the infrastructure + ansible.builtin.meta: refresh_inventory + +- name: Ensure inventory readiness + hosts: all gather_facts: no + tags: always tasks: - - name: Ensure the node is reachable + - name: Check if host is ready ansible.builtin.wait_for_connection: timeout: 60 -# Prepare and mount any attached volumes - name: Prepare and mount storage volumes - hosts: nodes + hosts: all gather_facts: no - become: True + become: yes + tags: infra tasks: - name: Prepare storage volumes - when: infra_type != "static" and storage_volumes | length > 0 + when: storage_volumes is defined and storage_volumes | length > 0 ansible.builtin.import_role: name: cloudera.exe.mount vars: mount_volumes: "{{ storage_volumes }}" mount_provider: "{{ infra_type }}" -# Prepare the FreeIPA sidecar services - name: Provision FreeIPA services hosts: freeipa gather_facts: yes become: yes + tags: freeipa vars: - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" - # If not set specifically: if AWS, then VPC CIDR + 2 (see https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) - resolv_nameservers: "{{ upstream_nameservers | default(default_dns[infra_type]) }}" - default_dns: - aws: "{{ [ hostvars.localhost.provision.vpc.cidr | ansible.utils.ipmath(2) ] }}" + vpc_cidr: "{{ lookup('cloud.terraform.tf_output', 'vpc', project_path='tf_proxied_cluster')['cidr_block'] }}" module_defaults: freeipa.ansible_freeipa.ipadnszone: ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" freeipa.ansible_freeipa.ipadnsrecord: ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" tasks: - - name: Set up the FreeIPA server + - name: Set up the local FreeIPA server ansible.builtin.import_role: name: cloudera.exe.freeipa_server vars: ipaserver_hostname: "{{ inventory_hostname }}" ipaserver_realm: "{{ freeipa.realm }}" - ipaserver_domain: "{{ name_prefix }}.{{ domain }}" + ipaserver_domain: "{{ domain }}" ipaserver_no_host_dns: yes ipaserver_setup_firewalld: no - ipaserver_setup_dns: "{{ enable_dns }}" - ipaserver_resolv_nameservers: "{{ resolv_nameservers }}" - ipaserver_auto_reverse: "{{ enable_dns | ternary(True, omit) }}" - ipaserver_no_forwarders: "{{ enable_dns | ternary(True, omit) }}" - ipaserver_forward_policy: "{{ enable_dns | ternary('only', omit) }}" - ipaserver_recursion_acl_cidr: "{{ enable_dns | ternary(hostvars.localhost.provision.vpc.cidr, omit) }}" + ipaserver_setup_dns: yes + # See https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html + ipaserver_resolv_nameservers: ["{{ vpc_cidr | ansible.utils.ipmath(2) }}"] + ipaserver_auto_reverse: yes + ipaserver_no_forwarders: yes + ipaserver_forward_policy: only + ipaserver_recursion_acl_cidr: "{{ vpc_cidr }}" ipaserver_copy_csr_to_controller: yes ipaclient_mkhomedir: yes - ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" # TODO Add test for these parameters - error in role is opaque + # TODO Add test for these parameters - error in role is opaque + ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" - - name: Configure provisioned FreeIPA global DNS - when: enable_dns - block: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in provisioned FreeIPA service - freeipa.ansible_freeipa.ipadnszone: - zone_name: "{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - forward_policy: none - - - name: Create reverse DNS zone for '{{ hostvars.localhost.provision.vpc.cidr }}' - freeipa.ansible_freeipa.ipadnszone: - name_from_ip: "{{ hostvars.localhost.provision.vpc.cidr }}" - dynamic_update: yes - allow_sync_ptr: yes - - - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' - when: "'cluster_ecs_masters' in groups" - block: - - name: Ensure DNS zone for ECS 'apps' - freeipa.ansible_freeipa.ipadnszone: - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - forward_policy: none - - - name: Ensure DNS wildcard records for ECS 'apps' and domain - freeipa.ansible_freeipa.ipadnsrecord: - name: "*" - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - record_type: "A" - record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" - create_reverse: no - - - name: Resolve provisioned FreeIPA DNS server addresses - ansible.builtin.set_fact: - dns_server_ips: "{{ dns_server_ips | default([]) | union([ ansible_default_ipv4['address'] ]) }}" - delegate_to: localhost - delegate_facts: true - -# Update existing FreeIPA DNS services -- name: Configure existing FreeIPA global DNS - hosts: localhost - connection: local - gather_facts: no - vars: - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" - dns_server: "{{ freeipa.server | default(lookup('ansible.builtin.env', 'IPA_HOST', default='Undefined')) }}" - module_defaults: - community.general.ipa_dnszone: - ipa_host: "{{ dns_server }}" - ipa_user: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default='Undefined')) }}" - ipa_pass: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" - tasks: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in existing FreeIPA service - when: enable_dns and dns_server is defined and 'freeipa' not in groups - block: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' - community.general.ipa_dnszone: - validate_certs: no - dynamicupdate: yes - allowsyncptr: yes - zone_name: "{{ name_prefix }}.{{ domain }}" - state: present - - - name: Create reverse DNS zone for '{{ infra_cidr }}' - community.general.ipa_dnszone: - validate_certs: no - dynamicupdate: yes - allowsyncptr: yes - zone_name: "{{ infra_cidr | ansible.utils.ipaddr('revdns') }}" - state: present - - - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' - when: "'cluster_ecs_masters' in groups" - block: - - name: Ensure DNS zone for ECS 'apps' - community.general.ipa_dnszone: - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - validate_certs: no - - - name: Ensure DNS wildcard records for ECS 'apps' and domain - community.general.ipa_dnszone: - name: "*" - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - record_type: "A" - record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" - validate_certs: no - - - name: Resolve existing FreeIPA DNS server addresses - ansible.builtin.set_fact: - dns_hosts: "{{ [ dns_server ] }}" - dns_server_ips: "{{ [ lookup('community.general.dig', dns_server ) ] }}" - -# Enroll deployment hosts with FreeIPA Kerberos and possibly with DNS -- name: Register hosts with FreeIPA - hosts: nodes:!freeipa + - name: Create DNS zone for '{{ domain }}' in provisioned FreeIPA service + freeipa.ansible_freeipa.ipadnszone: + zone_name: "{{ domain }}" + dynamic_update: yes + allow_sync_ptr: yes + forward_policy: none + + - name: Create reverse DNS zone for '{{ vpc_cidr }}' + freeipa.ansible_freeipa.ipadnszone: + name_from_ip: "{{ vpc_cidr }}" + dynamic_update: yes + allow_sync_ptr: yes + +- name: Register all hosts with local FreeIPA server + hosts: all:!freeipa gather_facts: yes - become: True + become: yes + tags: freeipa tasks: - - name: Register host with FreeIPA services - when: krb5_kdc_type == "Red Hat IPA" + - name: Register host with local FreeIPA server ansible.builtin.import_role: name: cloudera.exe.freeipa_client vars: - ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" + ipaserver_domain: "{{ domain }}" ipaserver_realm: "{{ freeipa.realm }}" - ipa_hosts: "{{ groups['freeipa'] | default(hostvars.localhost.dns_hosts) }}" - ipa_server_ips: "{{ hostvars.localhost.dns_server_ips }}" + ipa_hosts: "{{ groups['freeipa'] }}" + ipa_server_ips: "{{ groups['freeipa'] | map('extract', hostvars, ['ansible_host']) | list }}" ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" ipaadmin_principal: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default=omit)) }}" - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" + enable_dns: yes -# Prepare SME Lab resources -- name: Prepare TLS and supporting services resources - import_playbook: pre_setup_resources.yml - tags: - - prereq - - infra \ No newline at end of file +- name: Establish supporting services resources + hosts: deployment + gather_facts: no + become: yes + tags: prereq + tasks: + - name: Establish additional cluster host requirements + ansible.builtin.import_role: + name: cluster_reqs diff --git a/private-cloud/pvc-base-on-aws/pre_teardown.yml b/private-cloud/pvc-base-on-aws/pre_teardown.yml index 7ceedf0..c3c23a1 100644 --- a/private-cloud/pvc-base-on-aws/pre_teardown.yml +++ b/private-cloud/pvc-base-on-aws/pre_teardown.yml @@ -1,24 +1,31 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- - name: Deprovision the infrastructure resources hosts: localhost connection: local gather_facts: no - tags: - - teardown - - provision tasks: - - name: Deprovision the infrastructure resources in the cloud provider - when: groups['all'] | length > 0 - ansible.builtin.import_role: - name: cloudera.exe.provision - vars: - provision_state: absent - provision_provider: "{{ infra_type }}" - provision_inventory_file: inventory_static.ini - provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" - provision_state_storage: "{{ terraform.state.storage }}" - provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" - provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" - provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" - provision_name_prefix: "{{ name_prefix }}" + - name: Check for existing Terraform state file + ansible.builtin.stat: + path: tf_proxied_cluster/terraform.tfstate + register: __tf_state + + - name: Teardown the demo infrastructure via Terraform + when: __tf_state.stat.exists + cloud.terraform.terraform: + project_path: "tf_proxied_cluster/" + state: "absent" From b51f820b710cbbf017b403791335858d22f15dbf Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:06 -0500 Subject: [PATCH 35/45] Update config-template.yml and README for Terraform inventory execution Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/README.md | 73 +++++++++++-------- .../pvc-base-on-aws/config-template.yml | 30 +++++--- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/pvc-base-on-aws/README.md index 37d7ad7..0c2194e 100644 --- a/private-cloud/pvc-base-on-aws/README.md +++ b/private-cloud/pvc-base-on-aws/README.md @@ -13,8 +13,10 @@ To run, you need: * Docker (or a Docker alternative) -* AWS credentials (set via `AWS_PROFILE`) -* CDP Private Cloud Base license file credentials (set via `CDP_LICENSE_FILE`) +* `ansible-navigator` +* AWS credentials +* CDP Private Cloud Base license file +* SSH key(s) for bastion/jump host and cluster ### Configuration Variables @@ -27,25 +29,24 @@ Configuration is passed via environment variables and an user-facing configurati | Variable | Description | Status | |----------|-------------|--------| | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/example/.ssh/demo_ops.pub` | Mandatory | + | `SSH_PRIVATE_KEY_FILE` | | | | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/example/Documents/example_cloudera_license.txt` | Mandatory | | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | - | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. Used also for remote storage of Terraform state in AWS. | Mandatory | - -> **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. + | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. | Mandatory | #### Configuration file variables -Edit the `config.yml` user-facing configuration file to match your particular deployment. +Copy `config-template.yml` to `config.yml` and edit this user-facing configuration file to match your particular deployment. *NOTE:* `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. ```yaml -name_prefix: "labaw" # CHANGE THIS +name_prefix: "labaw" # CHANGE THIS owner_prefix: "pvc-base" owner_email: "example@cloudera.com" infra_region: "eu-west-1" -infra_type: "aws" # "aws", "static" +infra_type: "aws" # "aws", "static" domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) realm: "CLDR.EXAMPLE" # The Kerberos realm common_password: "Example776" @@ -64,6 +65,7 @@ deployment_tags: ### Pre-setup Playbook This definition-specific playbook includes tasks such as: + * Instructure provisioning * FreeIPA DNS and KRB services provisioning @@ -71,20 +73,21 @@ Run the following command ```bash ansible-navigator run pre_setup.yml \ --e @config.yml \ --e @definition.yml + -e @definition.yml \ + -e @config.yml ``` Once the pre-setup playbook completes confirm that: -* You can connect to each node via the inventory - see Confirm SSH Connectivity. Note that a A `validate_dns_lookups.yml` Playbook exists to check connectivity. -* Connect to FreeIPA UI and login with the `IPA_USER` and `IPA_PASSWORD` credentials in the configuration file. See Cluster Access for more details. +* You can connect to each node via the inventory - see [Confirm SSH Connectivity](#confirm-ssh-connectivity) for help. You can also run `ansible-navigator run validate_dns_lookups.yml` to check connectivity and DNS. +* Connect to FreeIPA UI and login with the `IPA_USER` and `IPA_PASSWORD` credentials in the configuration file. See [Cluster Access](#cluster-access) for details. ### Platform Playbooks -These playbooks configure and deploy PVC Base. They use the infrastructure provisioned (or assigned, if using `static` inventory). +These playbooks configure and deploy PVC Base. They use the infrastructure provisioned. Tasks include: + * System/host configuration * Cloudera Manager server and agent installation and configuration * Cluster template imports @@ -94,30 +97,30 @@ Run the following: ```bash # Run the 'external' system configuration ansible-navigator run external_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini + -e @definition.yml \ + -e @config.yml ``` ```bash # Run the 'internal' Cloudera installations and configurations ansible-navigator run internal_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini + -e @definition.yml \ + -e @config.yml ``` ```bash # Run the Cloudera cluster configuration and imports ansible-navigator run base_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini + -e @definition.yml \ + -e @config.yml ``` And lastly, the _postfix_: ```bash ansible-navigator run base_postfix.yml \ - -e @config.yml \ - -i inventory_static__aws.ini + -e @definition.yml \ + -e @config.yml ``` ## Cluster Access @@ -125,32 +128,40 @@ ansible-navigator run base_postfix.yml \ Once the cluster is up, you can access all of the UIs within, including the FreeIPA sidecar, via a SSH tunnel: ```bash -ssh -D -q -C -N @ +ssh -D 8157 -q -C -N ec2-user@ ``` -and use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega browser extension). -In the SOCKS5 proxy configuration, set Protocol to SOCKS5; Server to localhost and Port to 8157. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. +Use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega browser extension). + +In the SOCKS5 proxy configuration, set _Protocol_ to `SOCKS5`, _Server_ to `localhost`, and _Port_ to `8157`. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. -You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. +> **NOTE:** +> You will get a SSL warning for the self-signed certificate; this is expected given this particular definition as the local FreeIPA server has the root certificate. (You can always install this root certificate if you want to remove this notification!) In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! ```bash -ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @ +ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ec2-user@ ``` +> **NOTE:** +> The above assume you are using the default AMI image set in the Terraform configuration. If not, adjust the SSH user appropriately. + ## Teardown Run the following: ```bash ansible-navigator run pre_teardown.yml \ - -e @config.yml \ -e @definition.yml \ - -i inventory_static__aws.ini + -e @config.yml ``` -You can also run `terraform destroy` within the `tf_deployment_*` directory. +You can also run the direct Terraform command: + +```bash +ansible-navigator exec -- terraform -chdir=tf_proxied_cluster destroy -auto-approve +``` ## Troubleshooting @@ -159,7 +170,7 @@ You can also run `terraform destroy` within the `tf_deployment_*` directory. Run the following: ```bash -ansible -m ansible.builtin.ping -i inventory_static__aws.ini all +ansible-navigator exec -- ansible -m ansible.builtin.ping -i inventory.yml all ``` -This will check to see if the inventory file is well constructed, etc. +This will check to see if the inventory file is well constructed and the hosts are available via SSH. diff --git a/private-cloud/pvc-base-on-aws/config-template.yml b/private-cloud/pvc-base-on-aws/config-template.yml index b36be81..6041c57 100644 --- a/private-cloud/pvc-base-on-aws/config-template.yml +++ b/private-cloud/pvc-base-on-aws/config-template.yml @@ -1,18 +1,30 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- -name_prefix: "labaw" # CHANGE THIS +name_prefix: "pvclab" # CHANGE THIS owner_prefix: "pvc-base" owner_email: "example@cloudera.com" -infra_region: "eu-west-1" -infra_type: "aws" # "aws", "static" -domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) -realm: "CLDR.EXAMPLE" # The Kerberos realm -common_password: "Example776" -admin_password: "Example776" # "Main" default password +infra_region: "us-east-2" +domain: "{{ name_prefix }}.cldr.example" # The private, adhoc subdomain managed by the local FreeIPA server +realm: "CLDR.EXAMPLE" # The Kerberos realm managed by the local FreeIPA server +common_password: "Example776" # For external services, e.g. Postgres, TLS +admin_password: "Example776" # For CM-related services deployment_tags: owner: "{{ owner_prefix }}" email: "{{ owner_email }}" - project: "PvC Base Lab - {{ owner_prefix }}-{{ name_prefix }}" - enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" + project: "PVC Base - {{ owner_prefix }}-{{ name_prefix }}" deployment: "{{ name_prefix }}" deploy-tool: cloudera-deploy From 1a753bf2a8667c695e31821ae8bd1b03ba2fd64a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:06 -0500 Subject: [PATCH 36/45] Update cluster_reqs role for pre/post-distribution tasks Signed-off-by: Webster Mudge --- .../roles/cluster_reqs/tasks/RedHat-post.yml | 19 +++++++++++++ .../tasks/{Ubuntu.yml => Ubuntu-pre.yml} | 0 .../roles/cluster_reqs/tasks/main.yml | 27 ++++++++++++++----- .../roles/cluster_reqs/vars/RedHat.yml | 11 +++++--- 4 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml rename private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/{Ubuntu.yml => Ubuntu-pre.yml} (100%) diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml new file mode 100644 index 0000000..da61684 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml @@ -0,0 +1,19 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install required Python38 libraries + ansible.builtin.pip: + name: "{{ python38_libraries }}" + state: present + executable: pip3.8 diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu-pre.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu.yml rename to private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu-pre.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml index 1507c77..4c5dab2 100644 --- a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml @@ -29,23 +29,36 @@ - "{{ ansible_facts['os_family'] }}.yml" - "default.yml" -- name: Run distribution tasks +- name: Run distribution pre-tasks ansible.builtin.include_tasks: "{{ item }}" with_first_found: - - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" - - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" - - "{{ ansible_facts['distribution'] }}.yml" - - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" - - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" - - "{{ ansible_facts['os_family'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}-pre.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}-pre.yml" + - "{{ ansible_facts['distribution'] }}-pre.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}-pre.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}-pre.yml" + - "{{ ansible_facts['os_family'] }}-pre.yml" - "default.yml" - name: Install required system packages + when: system_packages ansible.builtin.package: name: "{{ system_packages }}" state: present - name: Install required Python libraries + when: python_libraries ansible.builtin.pip: name: "{{ python_libraries }}" state: present + +- name: Run distribution post-tasks + ansible.builtin.include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}-post.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}-post.yml" + - "{{ ansible_facts['distribution'] }}-post.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}-post.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}-post.yml" + - "{{ ansible_facts['os_family'] }}-post.yml" + - "default.yml" diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml index cb60e46..5a4f74f 100644 --- a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml +++ b/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml @@ -16,9 +16,12 @@ system_packages: - tree - jq - zip - - postgresql-devel - - gcc - - platform-python-devel + #- postgresql-devel + #- gcc + #- platform-python-devel + - python38 -python_libraries: +python_libraries: [] + +python38_libraries: - psycopg2-binary==2.9.5 From d2187b12b077c7ae2d69a62bac93589273978422 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:06 -0500 Subject: [PATCH 37/45] Update instance names for deployment Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/tf_bastion/main.tf | 2 +- private-cloud/pvc-base-on-aws/tf_bastion/variables.tf | 2 +- private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf index 147dfc2..7ab5f25 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/main.tf @@ -24,7 +24,7 @@ terraform { locals { security_group_name = var.security_group_name != "" ? var.security_group_name : "${var.prefix}-pvc-base-bastion" - instance_name = var.instance_name != "" ? var.instance_name : "${var.prefix}-pvc-base-bastion" + instance_name = var.name != "" ? var.name : "${var.prefix}-pvc-base-bastion" } data "aws_key_pair" "bastion" { diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf index 66b5f0b..002c59a 100644 --- a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf +++ b/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf @@ -54,7 +54,7 @@ variable "image_id" { description = "AMI image ID for the bastion" } -variable "instance_name" { +variable "name" { type = string description = "Name of the bastion instance" default = "" diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf index faa0568..31981fc 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf +++ b/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf @@ -115,6 +115,7 @@ module "bastion" { depends_on = [aws_key_pair.pvc_base, module.cluster_network] prefix = var.prefix + name = "${var.prefix}-bastion" image_id = data.aws_ami.pvc_base.image_id vpc_id = aws_vpc.pvc_base.id subnet_id = module.cluster_network.public_subnets[0].id @@ -160,7 +161,7 @@ module "masters" { depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] prefix = var.prefix - name = "${var.prefix}-pvc-base-master" + name = "${var.prefix}-master" image_id = data.aws_ami.pvc_base.image_id instance_type = "m5.4xlarge" ssh_key_pair = aws_key_pair.pvc_base.key_name @@ -178,7 +179,7 @@ module "workers" { depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] prefix = var.prefix - name = "${var.prefix}-pvc-base-worker" + name = "${var.prefix}-worker" quantity = 2 image_id = data.aws_ami.pvc_base.image_id instance_type = "c5.2xlarge" @@ -197,7 +198,7 @@ module "freeipa" { depends_on = [aws_key_pair.pvc_base, data.aws_ami.pvc_base] prefix = var.prefix - name = "${var.prefix}-pvc-base-freeipa" + name = "${var.prefix}-freeipa" image_id = data.aws_ami.pvc_base.image_id instance_type = "m5.large" # TODO Look up via region ssh_key_pair = aws_key_pair.pvc_base.key_name From e2d1fcaae0e055cb5e5fd86a3cbcadbe1e19b980 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:06 -0500 Subject: [PATCH 38/45] Remove IPA_USER and IPA_PASSWORD dependencies. Update KRB and LDAP parameters. Signed-off-by: Webster Mudge --- .../pvc-base-on-aws/ansible-navigator.yml | 4 +-- .../pvc-base-on-aws/config-template.yml | 21 ++++++--------- private-cloud/pvc-base-on-aws/definition.yml | 26 ++++++++++++------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml index 98fc070..400a074 100644 --- a/private-cloud/pvc-base-on-aws/ansible-navigator.yml +++ b/private-cloud/pvc-base-on-aws/ansible-navigator.yml @@ -40,8 +40,8 @@ ansible-navigator: - SSH_PUBLIC_KEY_FILE - SSH_PRIVATE_KEY_FILE - CDP_LICENSE_FILE - - IPA_USER - - IPA_PASSWORD + #- IPA_USER + #- IPA_PASSWORD set: ANSIBLE_SSH_CONTROL_PATH: "/dev/shm/cp%%h-%%p-%%r" ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" diff --git a/private-cloud/pvc-base-on-aws/config-template.yml b/private-cloud/pvc-base-on-aws/config-template.yml index 6041c57..dda9179 100644 --- a/private-cloud/pvc-base-on-aws/config-template.yml +++ b/private-cloud/pvc-base-on-aws/config-template.yml @@ -14,17 +14,12 @@ --- -name_prefix: "pvclab" # CHANGE THIS -owner_prefix: "pvc-base" -owner_email: "example@cloudera.com" -infra_region: "us-east-2" -domain: "{{ name_prefix }}.cldr.example" # The private, adhoc subdomain managed by the local FreeIPA server -realm: "CLDR.EXAMPLE" # The Kerberos realm managed by the local FreeIPA server -common_password: "Example776" # For external services, e.g. Postgres, TLS -admin_password: "Example776" # For CM-related services +name_prefix: "{{ mandatory }}" # Unique identifier for the deployment +infra_region: "us-east-2" +domain: "{{ name_prefix }}.cldr.example" # The deployment subdomain (private) managed by the local FreeIPA server +realm: "CLDR.DEPLOYMENT" # The Kerberos realm managed by the local FreeIPA server +common_password: "Example776" # For external services, e.g. FreeIPA, Postgres, TLS +admin_password: "Example776" # For Cloudera-related services, e.g. Cloudera Manager deployment_tags: - owner: "{{ owner_prefix }}" - email: "{{ owner_email }}" - project: "PVC Base - {{ owner_prefix }}-{{ name_prefix }}" - deployment: "{{ name_prefix }}" - deploy-tool: cloudera-deploy + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/pvc-base-on-aws/definition.yml index 9b081de..e652175 100644 --- a/private-cloud/pvc-base-on-aws/definition.yml +++ b/private-cloud/pvc-base-on-aws/definition.yml @@ -24,10 +24,10 @@ terraform: storage: local create_remote_storage: False s3_region: "{{ infra_region }}" - s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" + s3_bucket: "{{ name_prefix }}-tf-state" # SSH -public_key_id: "{{ owner_prefix }}-{{ name_prefix }}" +public_key_id: "{{ name_prefix }}-pvc-base" public_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" public_key_text: "{{ lookup('ansible.builtin.file', lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE')) }}" @@ -52,6 +52,8 @@ freeipa_enroll: no # FreeIPA realm settings freeipa: realm: "{{ realm }}" + ipaadmin_user: admin + ipaadmin_password: "{{ common_password }}" # Defer user and group creation to SSSD/FreeIPA registration # Use when you dont need ansible to run "ca_server" @@ -84,9 +86,9 @@ database_default_password: "{{ common_password }}" ### Red Hat FreeIPA krb5_kdc_type: Red Hat IPA -krb5_realm: "{{ realm }}" -krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user | default(lookup('env', 'IPA_USER', default='Undefined')) }}@{{ krb5_realm }}" -krb5_kdc_admin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" +krb5_realm: "{{ freeipa.realm }}" +krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user }}@{{ krb5_realm }}" +krb5_kdc_admin_password: "{{ freeipa.ipaadmin_password }}" krb5_enc_types: "aes256-cts aes128-cts" # TODO: This needs to either be explicit or rendered from a host with DNS resolution # Cannot rely on localhost to have DNS access, e.g. private networking on EC2 @@ -103,7 +105,7 @@ cloudera_manager_external_auth: auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" -base_dn: "dc={{ (krb5_realm | lower).split('.') | join(',dc=') }}" +base_dn: "dc={{ freeipa.realm.split('.') | map('lower') | join(',dc=') }}" user_dn: "cn=users,cn=accounts,{{ base_dn }}" group_dn: "cn=groups,cn=accounts,{{ base_dn }}" @@ -112,16 +114,20 @@ auth_providers: ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}:636" ldap_base_dn: "{{ base_dn }}" ldap_bind_user_dn: "uid=admin,{{ user_dn }}" - ldap_bind_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" + ldap_bind_password: "{{ freeipa.ipaadmin_password }}" ldap_search_base: user: "{{ user_dn }}" group: "{{ group_dn }}" ldap_object_class: user: "person" - group: "groupofnames" + group: "posixgroup" ldap_search_filter: - user: uid={0} - member: (member={0}) + user: uid + member: member + # group: (&(member={0})(objectclass=posixgroup)(!(cn=admins))) + ldap_attribute: + user: uid # Set to (uid={0}) + member: member # defaults to (member={0}) ldap_attr_user_name: "uid" ldap_attr_group_name: "cn" ldap_attr_group_membership: "member" From 9843819bcaa587c937505ed9ef64b4002f0c9d3b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:07 -0500 Subject: [PATCH 39/45] Add deployment summary artifacts Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/.gitignore | 4 ++ private-cloud/pvc-base-on-aws/pre_setup.yml | 2 +- private-cloud/pvc-base-on-aws/summary.yml | 34 +++++++++++++++ .../templates/deployment.md.j2 | 41 +++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 private-cloud/pvc-base-on-aws/summary.yml create mode 100644 private-cloud/pvc-base-on-aws/templates/deployment.md.j2 diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/pvc-base-on-aws/.gitignore index a925cf1..1040cfb 100644 --- a/private-cloud/pvc-base-on-aws/.gitignore +++ b/private-cloud/pvc-base-on-aws/.gitignore @@ -38,3 +38,7 @@ inventory_static* # Per-user configuration files config.yml + +# Per-deployment artifacts +*-DEPLOYMENT.md +*.ca.crt diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/pvc-base-on-aws/pre_setup.yml index ed322d1..8c06186 100644 --- a/private-cloud/pvc-base-on-aws/pre_setup.yml +++ b/private-cloud/pvc-base-on-aws/pre_setup.yml @@ -105,7 +105,7 @@ allow_sync_ptr: yes forward_policy: none - - name: Create reverse DNS zone for '{{ vpc_cidr }}' + - name: Create reverse DNS zone for '{{ vpc_cidr }}' in provisioned FreeIPA service freeipa.ansible_freeipa.ipadnszone: name_from_ip: "{{ vpc_cidr }}" dynamic_update: yes diff --git a/private-cloud/pvc-base-on-aws/summary.yml b/private-cloud/pvc-base-on-aws/summary.yml new file mode 100644 index 0000000..ba1c8d0 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/summary.yml @@ -0,0 +1,34 @@ +# Copyright 2023 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create a deployment summary document + hosts: localhost + connection: local + gather_facts: no + tasks: + - name: Create the deployment summary document + ansible.builtin.template: + src: "deployment.md.j2" + dest: "{{ name_prefix | upper }}-DEPLOYMENT.md" + +- name: Retrieve the FreeIPA CA certificate + hosts: freeipa + gather_facts: no + tasks: + - name: Retrieve the CA certificate + ansible.builtin.fetch: + src: "/etc/ipa/ca.crt" + dest: "{{ inventory_hostname }}.ca.crt" + flat: yes + run_once: yes diff --git a/private-cloud/pvc-base-on-aws/templates/deployment.md.j2 b/private-cloud/pvc-base-on-aws/templates/deployment.md.j2 new file mode 100644 index 0000000..da84903 --- /dev/null +++ b/private-cloud/pvc-base-on-aws/templates/deployment.md.j2 @@ -0,0 +1,41 @@ +# {{ name_prefix}} - PVC Base Deployment + +* **Prefix:** {{ name_prefix }} +* **Domain:** {{ domain }} +* **Realm:** {{ realm }} +* **SSH:** {{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') | basename }} + +## Deployment Access + +Enable the SSH tunnel to the deployment: + +```bash +ssh -D 8157 -qCN {{ groups['jump_host'] | map('extract', hostvars, 'ansible_user') | first }}@{{ groups['jump_host'] | map('extract', hostvars, 'ansible_host') | first }} +``` + +Use a SOCKS5 proxy switcher in your browser to access the deployment endpoints. +(An example is the _SwitchyOmega_ extension for Chrome-based browsers.) + +In the proxy configuration, set _Protocol_ to `SOCKS5`, _Server_ to +`localhost`, and _Port_ to `8157`. Ensure the SOCKS5 proxy is active when +clicking on the CDP Private Cloud Base UIs and endpoints that you wish to access. + +## SSL Certificate Installation + +Any SSL-enabled endpoint within the cluster will produce a SSL warning for the +self-signed certificate issued by the deployment's local FreeIPA server. If you +wish to avoid this warning, install the CA certificate, `{{ groups['freeipa'] | first }}.ca.crt`, +for your browser and/or system. + +## Endpoints + +* [Cloudera Manager](https://{{ groups['cloudera_manager'] | first }}:7183) +* [FreeIPA](https://{{ groups['freeipa'] | first }}) + +## Hosts + +|Host|FQDN| +|---|---| +{% for host in groups['all'] | sort | map('extract', hostvars) | list %} +| {{ host['inventory_hostname_short'] }} | {{ host['inventory_hostname'] }} | +{% endfor %} From df915a47a31307ded203ab0ae51651a858f27ab7 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:07 -0500 Subject: [PATCH 40/45] Update README for summary playbook and "all-in-one" option Signed-off-by: Webster Mudge --- private-cloud/pvc-base-on-aws/README.md | 56 ++++++++++++++----------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/pvc-base-on-aws/README.md index 0c2194e..2c463a1 100644 --- a/private-cloud/pvc-base-on-aws/README.md +++ b/private-cloud/pvc-base-on-aws/README.md @@ -29,39 +29,46 @@ Configuration is passed via environment variables and an user-facing configurati | Variable | Description | Status | |----------|-------------|--------| | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/example/.ssh/demo_ops.pub` | Mandatory | - | `SSH_PRIVATE_KEY_FILE` | | | + | `SSH_PRIVATE_KEY_FILE` | File path to the SSH private key. E.g. `/Users/example/.ssh/demo_ops` | Mandatory | | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/example/Documents/example_cloudera_license.txt` | Mandatory | - | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | - | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. | Mandatory | #### Configuration file variables Copy `config-template.yml` to `config.yml` and edit this user-facing configuration file to match your particular deployment. -*NOTE:* `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. +> [!IMPORTANT] +> `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. ```yaml -name_prefix: "labaw" # CHANGE THIS -owner_prefix: "pvc-base" -owner_email: "example@cloudera.com" -infra_region: "eu-west-1" -infra_type: "aws" # "aws", "static" -domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) -realm: "CLDR.EXAMPLE" # The Kerberos realm -common_password: "Example776" -admin_password: "Example776" +name_prefix: "{{ mandatory }}" # Unique identifier for the deployment +infra_region: "us-east-2" +domain: "{{ name_prefix }}.cldr.example" # The deployment subdomain +realm: "CLDR.DEPLOYMENT" # The Kerberos realm +common_password: "Example776" # For external services +admin_password: "Example776" # For Cloudera-related services deployment_tags: - owner: "{{ owner_prefix }}" - email: "{{ owner_email }}" - project: "PvC Base Lab - {{ owner_prefix }}-{{ name_prefix }}" - enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" - deployment: "{{ name_prefix }}" - deploy-tool: cloudera-deploy + deployment: "{{ name_prefix }}" + deploy-tool: cloudera-deploy ``` ## Execution +## All-in-One + +You can run all of the following steps at once, if you wish: + +```bash +ansible-navigator run \ + pre_setup.yml \ + external_setup.yml \ + internal_setup.yml \ + base_setup.yml \ + summary.yml \ + -e @definition.yml \ + -e @config.yml +``` + ### Pre-setup Playbook This definition-specific playbook includes tasks such as: @@ -115,10 +122,9 @@ ansible-navigator run base_setup.yml \ -e @config.yml ``` -And lastly, the _postfix_: - ```bash -ansible-navigator run base_postfix.yml \ +# Produce a deployment summary and retrieve the FreeIPA CA certificate +ansible-navigator run summary.yml \ -e @definition.yml \ -e @config.yml ``` @@ -135,8 +141,8 @@ Use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega brow In the SOCKS5 proxy configuration, set _Protocol_ to `SOCKS5`, _Server_ to `localhost`, and _Port_ to `8157`. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. -> **NOTE:** -> You will get a SSL warning for the self-signed certificate; this is expected given this particular definition as the local FreeIPA server has the root certificate. (You can always install this root certificate if you want to remove this notification!) +> [!CAUTION] +> You will get a SSL warning for the self-signed certificate; this is expected given this particular definition as the local FreeIPA server has no upstream certificates. However, you can install this CA certificate to remove this notification. In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! @@ -144,7 +150,7 @@ In addition, you can log into the jump host via SSH and get to any of the server ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ec2-user@ ``` -> **NOTE:** +> [!NOTE] > The above assume you are using the default AMI image set in the Terraform configuration. If not, adjust the SSH user appropriately. ## Teardown From a500b23da2387b7473a57827a8c14250116fceac Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 7 Dec 2023 15:31:07 -0500 Subject: [PATCH 41/45] Move to base/aws-iaas directory. Remove ecs-on-aws example (future work) Signed-off-by: Webster Mudge --- .../aws-iaas}/.gitignore | 0 .../aws-iaas}/README.md | 6 +- .../aws-iaas}/ansible-navigator.yml | 0 .../aws-iaas}/base_postfix.yml | 0 .../aws-iaas}/base_setup.yml | 0 .../aws-iaas}/cluster.yml | 0 .../aws-iaas}/config-template.yml | 0 .../aws-iaas}/definition.yml | 0 .../aws-iaas}/external_setup.yml | 0 .../aws-iaas}/group_vars/all.yml | 0 .../aws-iaas}/internal_setup.yml | 0 .../aws-iaas}/inventory.yml | 0 .../aws-iaas}/pre_setup.yml | 0 .../aws-iaas}/pre_teardown.yml | 0 .../aws-iaas}/roles/cluster_reqs/README.md | 0 .../roles/cluster_reqs/defaults/main.yml | 0 .../roles/cluster_reqs/meta/main.yml | 0 .../roles/cluster_reqs/tasks/RedHat-post.yml | 0 .../roles/cluster_reqs/tasks/Ubuntu-pre.yml | 0 .../roles/cluster_reqs/tasks/default.yml | 0 .../roles/cluster_reqs/tasks/main.yml | 0 .../roles/cluster_reqs/tests/inventory | 0 .../roles/cluster_reqs/tests/test.yml | 0 .../roles/cluster_reqs/vars/RedHat.yml | 0 .../roles/cluster_reqs/vars/Ubuntu.yml | 0 .../roles/cluster_reqs/vars/default.yml | 0 .../roles/cluster_reqs/vars/main.yml | 0 .../aws-iaas}/summary.yml | 0 .../aws-iaas}/templates/deployment.md.j2 | 0 .../aws-iaas}/templates/infra.tfvars.j2 | 0 .../aws-iaas}/tf_bastion/main.tf | 0 .../aws-iaas}/tf_bastion/outputs.tf | 0 .../aws-iaas}/tf_bastion/variables.tf | 0 .../aws-iaas}/tf_hosts/main.tf | 0 .../aws-iaas}/tf_hosts/outputs.tf | 0 .../aws-iaas}/tf_hosts/variables.tf | 0 .../aws-iaas}/tf_network/main.tf | 0 .../aws-iaas}/tf_network/outputs.tf | 0 .../aws-iaas}/tf_network/variables.tf | 0 .../aws-iaas}/tf_proxied_cluster/main.tf | 2 +- .../aws-iaas}/tf_proxied_cluster/outputs.tf | 0 .../aws-iaas}/tf_proxied_cluster/variables.tf | 4 +- .../aws-iaas}/validate_dns_lookups.yml | 0 private-cloud/ecs-on-aws/.gitignore | 17 -- private-cloud/ecs-on-aws/README.md | 165 ----------- .../ecs-on-aws/ansible-navigator.yml | 49 --- private-cloud/ecs-on-aws/base_postfix.yml | 38 --- private-cloud/ecs-on-aws/base_setup.yml | 38 --- private-cloud/ecs-on-aws/cluster.yml | 279 ------------------ private-cloud/ecs-on-aws/config.yml | 18 -- private-cloud/ecs-on-aws/definition.yml | 245 --------------- private-cloud/ecs-on-aws/external_setup.yml | 48 --- private-cloud/ecs-on-aws/group_vars/all.yml | 3 - private-cloud/ecs-on-aws/hostvars.j2 | 9 - private-cloud/ecs-on-aws/instance_vars.j2 | 13 - private-cloud/ecs-on-aws/internal_setup.yml | 38 --- private-cloud/ecs-on-aws/pre_setup.yml | 231 --------------- .../ecs-on-aws/pre_setup_resources.yml | 24 -- private-cloud/ecs-on-aws/pre_teardown.yml | 24 -- .../ecs-on-aws/validate_dns_lookups.yml | 27 -- 60 files changed, 6 insertions(+), 1272 deletions(-) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/.gitignore (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/README.md (96%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/ansible-navigator.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/base_postfix.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/base_setup.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/cluster.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/config-template.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/definition.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/external_setup.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/group_vars/all.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/internal_setup.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/inventory.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/pre_setup.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/pre_teardown.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/README.md (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/defaults/main.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/meta/main.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tasks/RedHat-post.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tasks/Ubuntu-pre.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tasks/default.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tasks/main.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tests/inventory (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/tests/test.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/vars/RedHat.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/vars/Ubuntu.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/vars/default.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/roles/cluster_reqs/vars/main.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/summary.yml (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/templates/deployment.md.j2 (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/templates/infra.tfvars.j2 (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_bastion/main.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_bastion/outputs.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_bastion/variables.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_hosts/main.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_hosts/outputs.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_hosts/variables.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_network/main.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_network/outputs.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_network/variables.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_proxied_cluster/main.tf (99%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_proxied_cluster/outputs.tf (100%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/tf_proxied_cluster/variables.tf (97%) rename private-cloud/{pvc-base-on-aws => base/aws-iaas}/validate_dns_lookups.yml (100%) delete mode 100644 private-cloud/ecs-on-aws/.gitignore delete mode 100644 private-cloud/ecs-on-aws/README.md delete mode 100644 private-cloud/ecs-on-aws/ansible-navigator.yml delete mode 100644 private-cloud/ecs-on-aws/base_postfix.yml delete mode 100644 private-cloud/ecs-on-aws/base_setup.yml delete mode 100644 private-cloud/ecs-on-aws/cluster.yml delete mode 100644 private-cloud/ecs-on-aws/config.yml delete mode 100644 private-cloud/ecs-on-aws/definition.yml delete mode 100644 private-cloud/ecs-on-aws/external_setup.yml delete mode 100644 private-cloud/ecs-on-aws/group_vars/all.yml delete mode 100644 private-cloud/ecs-on-aws/hostvars.j2 delete mode 100644 private-cloud/ecs-on-aws/instance_vars.j2 delete mode 100644 private-cloud/ecs-on-aws/internal_setup.yml delete mode 100644 private-cloud/ecs-on-aws/pre_setup.yml delete mode 100644 private-cloud/ecs-on-aws/pre_setup_resources.yml delete mode 100644 private-cloud/ecs-on-aws/pre_teardown.yml delete mode 100644 private-cloud/ecs-on-aws/validate_dns_lookups.yml diff --git a/private-cloud/pvc-base-on-aws/.gitignore b/private-cloud/base/aws-iaas/.gitignore similarity index 100% rename from private-cloud/pvc-base-on-aws/.gitignore rename to private-cloud/base/aws-iaas/.gitignore diff --git a/private-cloud/pvc-base-on-aws/README.md b/private-cloud/base/aws-iaas/README.md similarity index 96% rename from private-cloud/pvc-base-on-aws/README.md rename to private-cloud/base/aws-iaas/README.md index 2c463a1..224c565 100644 --- a/private-cloud/pvc-base-on-aws/README.md +++ b/private-cloud/base/aws-iaas/README.md @@ -1,6 +1,6 @@ -# PvC Base Cluster on AWS +# PvC Base Cluster on AWS IaaS -> Constructs a CDP Private Cloud Base cluster running on AWS. +> Constructs a CDP Private Cloud Base cluster running on AWS IaaS. ## Known Issues @@ -20,7 +20,7 @@ To run, you need: ### Configuration Variables -Configuration is passed via environment variables and an user-facing configuration file. +Configuration is passed via environment variables and an user-managed configuration file. #### Environment Variables diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/base/aws-iaas/ansible-navigator.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/ansible-navigator.yml rename to private-cloud/base/aws-iaas/ansible-navigator.yml diff --git a/private-cloud/pvc-base-on-aws/base_postfix.yml b/private-cloud/base/aws-iaas/base_postfix.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/base_postfix.yml rename to private-cloud/base/aws-iaas/base_postfix.yml diff --git a/private-cloud/pvc-base-on-aws/base_setup.yml b/private-cloud/base/aws-iaas/base_setup.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/base_setup.yml rename to private-cloud/base/aws-iaas/base_setup.yml diff --git a/private-cloud/pvc-base-on-aws/cluster.yml b/private-cloud/base/aws-iaas/cluster.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/cluster.yml rename to private-cloud/base/aws-iaas/cluster.yml diff --git a/private-cloud/pvc-base-on-aws/config-template.yml b/private-cloud/base/aws-iaas/config-template.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/config-template.yml rename to private-cloud/base/aws-iaas/config-template.yml diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/base/aws-iaas/definition.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/definition.yml rename to private-cloud/base/aws-iaas/definition.yml diff --git a/private-cloud/pvc-base-on-aws/external_setup.yml b/private-cloud/base/aws-iaas/external_setup.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/external_setup.yml rename to private-cloud/base/aws-iaas/external_setup.yml diff --git a/private-cloud/pvc-base-on-aws/group_vars/all.yml b/private-cloud/base/aws-iaas/group_vars/all.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/group_vars/all.yml rename to private-cloud/base/aws-iaas/group_vars/all.yml diff --git a/private-cloud/pvc-base-on-aws/internal_setup.yml b/private-cloud/base/aws-iaas/internal_setup.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/internal_setup.yml rename to private-cloud/base/aws-iaas/internal_setup.yml diff --git a/private-cloud/pvc-base-on-aws/inventory.yml b/private-cloud/base/aws-iaas/inventory.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/inventory.yml rename to private-cloud/base/aws-iaas/inventory.yml diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/base/aws-iaas/pre_setup.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/pre_setup.yml rename to private-cloud/base/aws-iaas/pre_setup.yml diff --git a/private-cloud/pvc-base-on-aws/pre_teardown.yml b/private-cloud/base/aws-iaas/pre_teardown.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/pre_teardown.yml rename to private-cloud/base/aws-iaas/pre_teardown.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md b/private-cloud/base/aws-iaas/roles/cluster_reqs/README.md similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/README.md rename to private-cloud/base/aws-iaas/roles/cluster_reqs/README.md diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/defaults/main.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/defaults/main.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/defaults/main.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/meta/main.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/meta/main.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/meta/main.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/RedHat-post.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/RedHat-post.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/RedHat-post.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu-pre.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/Ubuntu-pre.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/Ubuntu-pre.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/Ubuntu-pre.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/default.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/default.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/default.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/main.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tasks/main.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tasks/main.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory b/private-cloud/base/aws-iaas/roles/cluster_reqs/tests/inventory similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/inventory rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tests/inventory diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/tests/test.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/tests/test.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/tests/test.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/vars/RedHat.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/RedHat.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/vars/RedHat.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/vars/Ubuntu.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/Ubuntu.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/vars/Ubuntu.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/vars/default.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/default.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/vars/default.yml diff --git a/private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml b/private-cloud/base/aws-iaas/roles/cluster_reqs/vars/main.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/roles/cluster_reqs/vars/main.yml rename to private-cloud/base/aws-iaas/roles/cluster_reqs/vars/main.yml diff --git a/private-cloud/pvc-base-on-aws/summary.yml b/private-cloud/base/aws-iaas/summary.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/summary.yml rename to private-cloud/base/aws-iaas/summary.yml diff --git a/private-cloud/pvc-base-on-aws/templates/deployment.md.j2 b/private-cloud/base/aws-iaas/templates/deployment.md.j2 similarity index 100% rename from private-cloud/pvc-base-on-aws/templates/deployment.md.j2 rename to private-cloud/base/aws-iaas/templates/deployment.md.j2 diff --git a/private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 b/private-cloud/base/aws-iaas/templates/infra.tfvars.j2 similarity index 100% rename from private-cloud/pvc-base-on-aws/templates/infra.tfvars.j2 rename to private-cloud/base/aws-iaas/templates/infra.tfvars.j2 diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/main.tf b/private-cloud/base/aws-iaas/tf_bastion/main.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_bastion/main.tf rename to private-cloud/base/aws-iaas/tf_bastion/main.tf diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf b/private-cloud/base/aws-iaas/tf_bastion/outputs.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_bastion/outputs.tf rename to private-cloud/base/aws-iaas/tf_bastion/outputs.tf diff --git a/private-cloud/pvc-base-on-aws/tf_bastion/variables.tf b/private-cloud/base/aws-iaas/tf_bastion/variables.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_bastion/variables.tf rename to private-cloud/base/aws-iaas/tf_bastion/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/main.tf b/private-cloud/base/aws-iaas/tf_hosts/main.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_hosts/main.tf rename to private-cloud/base/aws-iaas/tf_hosts/main.tf diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf b/private-cloud/base/aws-iaas/tf_hosts/outputs.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_hosts/outputs.tf rename to private-cloud/base/aws-iaas/tf_hosts/outputs.tf diff --git a/private-cloud/pvc-base-on-aws/tf_hosts/variables.tf b/private-cloud/base/aws-iaas/tf_hosts/variables.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_hosts/variables.tf rename to private-cloud/base/aws-iaas/tf_hosts/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_network/main.tf b/private-cloud/base/aws-iaas/tf_network/main.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_network/main.tf rename to private-cloud/base/aws-iaas/tf_network/main.tf diff --git a/private-cloud/pvc-base-on-aws/tf_network/outputs.tf b/private-cloud/base/aws-iaas/tf_network/outputs.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_network/outputs.tf rename to private-cloud/base/aws-iaas/tf_network/outputs.tf diff --git a/private-cloud/pvc-base-on-aws/tf_network/variables.tf b/private-cloud/base/aws-iaas/tf_network/variables.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_network/variables.tf rename to private-cloud/base/aws-iaas/tf_network/variables.tf diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf b/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf similarity index 99% rename from private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf rename to private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf index 31981fc..723c104 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/main.tf +++ b/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf @@ -43,7 +43,7 @@ locals { name = ["RHEL-8.6*"] architecture = ["x86_64"] } - domain = var.domain != "" ? var.domain : "${var.prefix}.pvc-base.cldr.example" + domain = var.domain != "" ? var.domain : "${var.prefix}.pvc-base.cldr.example" vpc_name = var.vpc_name != "" ? var.vpc_name : "${var.prefix}-pvc-base" igw_name = var.igw_name != "" ? var.igw_name : "${var.prefix}-pvc-base-igw" } diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf b/private-cloud/base/aws-iaas/tf_proxied_cluster/outputs.tf similarity index 100% rename from private-cloud/pvc-base-on-aws/tf_proxied_cluster/outputs.tf rename to private-cloud/base/aws-iaas/tf_proxied_cluster/outputs.tf diff --git a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf b/private-cloud/base/aws-iaas/tf_proxied_cluster/variables.tf similarity index 97% rename from private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf rename to private-cloud/base/aws-iaas/tf_proxied_cluster/variables.tf index baecd9e..1a8187f 100644 --- a/private-cloud/pvc-base-on-aws/tf_proxied_cluster/variables.tf +++ b/private-cloud/base/aws-iaas/tf_proxied_cluster/variables.tf @@ -62,7 +62,7 @@ variable "igw_name" { } variable "domain" { - type = string + type = string description = "Private subdomain for proxied hosts, e.g. pvc-base.cldr.example" - default = "" + default = "" } diff --git a/private-cloud/pvc-base-on-aws/validate_dns_lookups.yml b/private-cloud/base/aws-iaas/validate_dns_lookups.yml similarity index 100% rename from private-cloud/pvc-base-on-aws/validate_dns_lookups.yml rename to private-cloud/base/aws-iaas/validate_dns_lookups.yml diff --git a/private-cloud/ecs-on-aws/.gitignore b/private-cloud/ecs-on-aws/.gitignore deleted file mode 100644 index e5a1363..0000000 --- a/private-cloud/ecs-on-aws/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -# Ansible Navigator assets -ansible-navigator.log -runs -context - -# Terraform deployments -tf_deployment* - -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Static inventory files -inventory_static* \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/README.md b/private-cloud/ecs-on-aws/README.md deleted file mode 100644 index 0b8c7b5..0000000 --- a/private-cloud/ecs-on-aws/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# PvC ECS Cluster on AWS - -> Constructs a CDP Private Cloud Base cluster suitable for enablement in configuring and using Ozone on a Private Cloud ECS Data Services (DS) cluster. - -## Known Issues - -| Issue | Description | Workaround | -|-------|-------------|------------| -| Cluster instances unavailable after the `external_setup.yml` Playbook | The cluster EC2 instances become unavailable after the `external_setup.yml` Playbook. During subsequent playbooks the hosts becomes unreachable and in the EC2 console the VM instances fail the reachability health check. | Restart the EC2 instances via the console. | - -## Requirements - -To run, you need: - -* Docker (or a Docker alternative) -* AWS credentials (set via `AWS_PROFILE`) -* CDP Private Cloud Base license file credentials (set via `CDP_LICENSE_FILE`) - -### Configuration Variables - -Configuration is passed via environment variables and an user-facing configuration file. - -#### Environment Variables - -* Set up the following definition environment variables: - - | Variable | Description | Status | - |----------|-------------|--------| - | `SSH_PUBLIC_KEY_FILE` | File path to the SSH public key that will be uploaded to the cloud provider (using the `name_prefix` variable as the key label). E.g. `/Users/example/.ssh/demo_ops.pub` | Mandatory | - | `CDP_LICENSE_FILE` | File path to a CDP Private Cloud Base license. E.g. `/Users/example/Documents/example_cloudera_license.txt` | Mandatory | - | `IPA_USER` | Set this to `admin`. The adminstrator user for FreeIPA. | Mandatory | - | `IPA_PASSWORD` | The adminstrator and directory password for FreeIPA | Mandatory | - | `AWS_PROFILE` | The profile label for your AWS credentials. Otherwise, use the associated `AWS_*` parameters. Used also for remote storage of Terraform state in AWS. | Mandatory | - -> **_NOTE:_** For OSX, set the following: `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES` to allow the WinRM modules to function. - -#### Configuration file variables - -Edit the `config.yml` user-facing configuration file to match your particular deployment. - -*NOTE:* `name_prefix` should be 4-7 characters and is the "primary key" for the deployment. `owner_prefix` is used in circumstances to differentiate resources such as the SSH key label in the cloud provider and the subdomain(s) for the private DNS service. - -```yaml -name_prefix: "labaw" # CHANGE THIS -owner_prefix: "pvc-ecs" -owner_email: "example@cloudera.com" -infra_region: "eu-west-1" -infra_type: "aws" # "aws", "static" -domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) -realm: "CLDR.EXAMPLE" # The Kerberos realm -common_password: "Example776" -admin_password: "Example776" -deployment_tags: - owner: "{{ owner_prefix }}" - email: "{{ owner_email }}" - project: "PvC ECS Lab - {{ owner_prefix }}-{{ name_prefix }}" - enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" - deployment: "{{ name_prefix }}" - deploy-tool: cloudera-deploy -``` - -## Execution - -### Pre-setup Playbook - -This definition-specific playbook includes tasks such as: -* Instructure provisioning -* FreeIPA DNS and KRB services provisioning - -Run the following command - -```bash -ansible-navigator run pre_setup.yml \ --e @config.yml \ --e @definition.yml -``` - -Once the pre-setup playbook completes confirm that: - -* You can connect to each node via the inventory - see Confirm SSH Connectivity. Note that a A `validate_dns_lookups.yml` Playbook exists to check connectivity. -* Connect to FreeIPA UI and login with the `IPA_USER` and `IPA_PASSWORD` credentials in the configuration file. See Cluster Access for more details. - -### Platform Playbooks - -These playbooks configure and deploy PVC Base and, optionally, ECS. They use the infrastructure provisioned (or assigned, if using `static` inventory). - -Tasks include: -* System/host configuration -* Cloudera Manager server and agent installation and configuration -* Cluster template imports - -Run the following: - -```bash -# Run the 'external' system configuration -ansible-navigator run external_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini -``` - -```bash -# Run the 'internal' Cloudera installations and configurations -ansible-navigator run internal_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini -``` - -```bash -# Run the Cloudera cluster configuration and imports -ansible-navigator run base_setup.yml \ - -e @config.yml \ - -i inventory_static__aws.ini -``` - -And lastly, the _postfix_: - -```bash -ansible-navigator run base_postfix.yml \ - -e @config.yml \ - -i inventory_static__aws.ini -``` - -## Cluster Access - -Once the cluster is up, you can access all of the UIs within, including the FreeIPA sidecar, via a SSH tunnel: - -```bash -ssh -D -q -C -N @ -``` - -and use a SOCKS5 proxy switcher in your browser (an example is the SwitchyOmega browser extension). -In the SOCKS5 proxy configuration, set Protocol to SOCKS5; Server to localhost and Port to 8157. Ensure the SOCKS5 proxy is active when clicking on the CDP UI that you wish to access. - -You will get a SSL warning for the self-signed certificate; this is expected given this particular definition. - -In addition, you can log into the jump host via SSH and get to any of the servers within the cluster. Remember to forward your SSH key! - -```bash -ssh -A -C -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @ -``` - -## Teardown - -Run the following: - -```bash -ansible-navigator run pre_teardown.yml \ - -e @config.yml \ - -e @definition.yml \ - -i inventory_static__aws.ini -``` - -You can also run `terraform destroy` within the `tf_deployment_*` directory. - -## Troubleshooting - -### Confirm SSH Connectivity - -Run the following: - -```bash -ansible -m ansible.builtin.ping -i inventory_static__aws.ini all -``` - -This will check to see if the inventory file is well constructed, etc. diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml deleted file mode 100644 index a83766b..0000000 --- a/private-cloud/ecs-on-aws/ansible-navigator.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -ansible-navigator: - playbook-artifact: - save-as: "runs/{playbook_name}-{time_stamp}.json" - - ansible-runner: - artifact-dir: runs - rotate-artifacts-count: 3 - - logging: - level: debug - append: False - - execution-environment: - container-engine: docker - enabled: True - environment-variables: - pass: - - AWS_PROFILE - - CDP_PROFILE - - SSH_PUBLIC_KEY_FILE - - CDP_LICENSE_FILE - - IPA_USER - - IPA_PASSWORD - set: - ANSIBLE_SSH_CONTROL_PATH: "/dev/shm/cp%%h-%%p-%%r" - ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" - ANSIBLE_GATHERING: "smart" - ANSIBLE_DEPRECATION_WARNINGS: False - ANSIBLE_HOST_KEY_CHECKING: False - ANSIBLE_SSH_RETRIES: 10 - image: ghcr.io/cloudera-labs/cldr-runner:aws-latest - pull: - arguments: - - "--tls-verify=false" - volume-mounts: - - src: "${SSH_PUBLIC_KEY_FILE}" - dest: "${SSH_PUBLIC_KEY_FILE}" - - src: "${CDP_LICENSE_FILE}" - dest: "${CDP_LICENSE_FILE}" - - src: "~/.aws" - dest: "/runner/.aws" - options: "Z" - - src: "~/.ssh" - dest: "/runner/.ssh" - options: "Z" - container-options: - - "--network=host" diff --git a/private-cloud/ecs-on-aws/base_postfix.yml b/private-cloud/ecs-on-aws/base_postfix.yml deleted file mode 100644 index c7336f5..0000000 --- a/private-cloud/ecs-on-aws/base_postfix.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: Postfix CDP Private Cloud clusters - hosts: localhost - connection: local - gather_facts: yes - vars: - definition_path: "./" - tasks: - - name: Set of deployment variables from definition.yml - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always - -- name: Postfix clusters for CDP Private Cloud - ansible.builtin.import_playbook: cloudera.exe.pvc_base_postfix.yml diff --git a/private-cloud/ecs-on-aws/base_setup.yml b/private-cloud/ecs-on-aws/base_setup.yml deleted file mode 100644 index be6790b..0000000 --- a/private-cloud/ecs-on-aws/base_setup.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: Set up CDP Private Cloud clusters - hosts: localhost - connection: local - gather_facts: yes - vars: - definition_path: "./" - tasks: - - name: Set of deployment variables from definition.yml - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always - -- name: Set up clusters for CDP Private Cloud - ansible.builtin.import_playbook: cloudera.exe.pvc_base_setup.yml diff --git a/private-cloud/ecs-on-aws/cluster.yml b/private-cloud/ecs-on-aws/cluster.yml deleted file mode 100644 index 622afaa..0000000 --- a/private-cloud/ecs-on-aws/cluster.yml +++ /dev/null @@ -1,279 +0,0 @@ ---- - -cluster_name_base: PVC-Base - -clusters: - - name: PVC-ECS - type: ecs - pvc_repository: "{{ ecs_options.repository }}" - application_domain: "{{ name_prefix }}.{{ domain }}" - datalake: "{{ cluster_name_base }}" - ecs_database_mode: embedded - services: [ DOCKER, ECS ] - security: - tls: true - repositories: "{{ ecs_options.parcel_repositories }}" - configs: - DOCKER: - SERVICEWIDE: - defaultDataPath: "/mnt/docker" - docker_images_destination_registry_user: registry-user - ECS: - SERVICEWIDE: - docker: docker - cp_prometheus_ingress_user: cloudera-manager - cp_prometheus_ingress_password: "{{ admin_password }}" - infra_prometheus_ingress_user: cloudera-manager - infra_prometheus_ingress_password: "{{ admin_password }}" - longhorn_replication: 2 - lsoDataPath: "/ecs/local" - defaultDataPath: "/ecs/longhorn-storage" - nfs_over_provisioning: 800 - # TODO: Create cert and key for ECS ingress controller - # ssl_certificate: /opt/cloudera/security/pki/ecs.pem - # ssl_private_key: /opt/cloudera/security/pki/ecs.key - host_templates: - ECSMaster: - DOCKER: [ DOCKER_SERVER ] - ECS: [ ECS_SERVER ] - ECSWorker: - DOCKER: [ DOCKER_SERVER ] - ECS: [ ECS_AGENT ] - controlplane_config: - ContainerInfo: - Mode: public - CopyDocker: false - Database: - Mode: embedded - EmbeddedDbStorage: 50 - - - name: PVC-Base - type: base - services: [ATLAS, HBASE, HDFS, HIVE, HIVE_ON_TEZ, INFRA_SOLR, KAFKA, OZONE, RANGER, SPARK_ON_YARN, TEZ, YARN, ZOOKEEPER] - security: - kerberos: true - tls: true - repositories: "{{ repositories }}" - configs: - ATLAS: - SERVICEWIDE: - kerberos.auth.enable: true - ATLAS_SERVER: - atlas_authentication_method_file: true - atlas_admin_password: "{{ common_password }}" - atlas_sso_knox_enabled: false - atlas_authentication_method_trustedproxy: false - atlas_authentication_method_pam: false - atlas_authentication_method_ldap: true - atlas_authentication_method_ldap_url: "{{ auth_provider.ldap_url }}" - atlas_authentication_method_ldap_bind_dn: "{{ auth_provider.ldap_bind_user_dn }}" - atlas_authentication_method_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" - atlas_authentication_method_ldap_userDNpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" - atlas_authentication_method_ldap_base_dn: "{{ auth_provider.ldap_search_base.user }}" - atlas_authentication_method_ldap_user_searchfilter: "{{ auth_provider.ldap_search_filter.user }}" - atlas_authentication_method_ldap_groupSearchBase: "{{ auth_provider.ldap_search_base.group }}" - atlas_authentication_method_ldap_groupSearchFilter: "{{ auth_provider.ldap_search_filter.member }}" - atlas_authentication_method_ldap_groupRoleAttribute: "{{ auth_provider.ldap_attr_group_name }}" - atlas_authentication_method_ldap_type: "ldap" - atlas_authentication_method_ldap_default_role: "ROLE_ADMIN" - HBASE: - SERVICEWIDE: - hadoop_secure_web_ui: true - hbase_graceful_stop_timeout: 5 - REGIONSERVER: - hbase_regionserver_java_heapsize: 268435456 - MASTER: - hbase_master_java_heapsize: 134217728 - GATEWAY: - hbase_client_java_heapsize: 134217728 - # NOTE: Have removed /mnt from the various HFDS directories - HDFS: - SERVICEWIDE: - hdfs_verify_ec_with_topology_enabled: false - enable_ranger_authorization: true - hadoop_secure_web_ui: false - trusted_realms: "{{ realm | upper }}, {{ realm | lower }}" - DATANODE: - datanode_java_heapsize: 134217728 - dfs_data_dir_list: /dfs/dn - dfs_datanode_max_locked_memory: 0 - dfs_datanode_failed_volumes_tolerated: 0 - GATEWAY: - hdfs_client_java_heapsize: 134217728 - NAMENODE: - dfs_name_dir_list: /dfs/nn - namenode_java_heapsize: 134217728 - role_config_suppression_namenode_java_heapsize_minimum_validator: true - SECONDARYNAMENODE: - secondary_namenode_java_heapsize: 134217728 - fs_checkpoint_dir_list: /dfs/snn - HIVE: - SERVICEWIDE: - ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" - GATEWAY: - hive_client_java_heapsize: 134217728 - HIVEMETASTORE: - hive_metastore_java_heapsize: 268435456 - metastore_canary_health_enabled: false - HIVE_ON_TEZ: - SERVICEWIDE: - ranger_plugin_urlauth_filesystem_schemes: "file:,wasb:,adl:" - KAFKA: - SERVICEWIDE: - kerberos.auth.enable: true - offsets.topic.replication.factor: 1 - transaction.state.log.replication.factor: 1 - transaction.state.log.min.isr: 1 - producer.metrics.enable: false - service_config_suppression_kafka_broker_count_validator: true - service_config_suppression_offsets.topic.replication.factor: true - service_config_suppression_transaction.state.log.replication.factor: true - KAFKA_BROKER: - graceful_stop_timeout: 5 - OZONE: - SERVICEWIDE: - ozone.security.enabled: true - ozone.service.id: ozone1 - ozone.security.http.kerberos.enabled: false - service_config_suppression_ozone_manager_count_validator: true - service_config_suppression_storage_container_manager_count_validator: true - OZONE_MANAGER: - om_max_heap_size: 2048 - STORAGE_CONTAINER_MANAGER: - scm_max_heap_size: 2048 - OZONE_DATANODE: - ozone-conf/ozone-site.xml_role_safety_valve: | - - hdds.datanode.client.port - 9874 - - ozone_datanode_heap_size: 2048 - OZONE_PROMETHEUS: - ozone.prometheus.http-port: 19090 - RANGER: - SERVICEWIDE: - keyadmin_user_password: "{{ common_password }}" - rangeradmin_user_password: "{{ common_password }}" - rangertagsync_user_password: "{{ common_password }}" - rangerusersync_user_password: "{{ common_password }}" - RANGER_ADMIN: - ranger_admin_max_heap_size: 1024 - ranger_authentication_method: "LDAP" - ranger.ldap.url: "{{ auth_provider.ldap_url }}" - ranger.ldap.bind.dn: "{{ auth_provider.ldap_bind_user_dn }}" - ranger_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" - ranger.ldap.user.dnpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" - ranger.ldap.base.dn: "{{ auth_provider.ldap_base_dn }}" - ranger.ldap.user.searchfilter: "(&(sAMAccountName={0})(objectClass=person))" - ranger.ldap.group.searchfilter: "(&(member={0})(objectClass=group))" - ranger.ldap.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" - ranger.ldap.group.roleattribute: "{{ auth_provider.ldap_attr_group_name }}" - ranger.ldap.referral: "follow" - RANGER_TAGSYNC: - ranger_tagsync_max_heap_size: 1024 - RANGER_USERSYNC: - ranger_usersync_max_heap_size: 1024 - ranger.usersync.ldap.binddn: "{{ auth_provider.ldap_bind_user_dn }}" - ranger.usersync.ldap.url: "{{ auth_provider.ldap_url }}" - ranger_usersync_ldap_ldapbindpassword: "{{ auth_provider.ldap_bind_password }}" - ranger.usersync.ldap.referral: follow - ranger.usersync.ldap.username.caseconversion: lower - ranger.usersync.ldap.groupname.caseconversion: lower - ranger.usersync.ldap.grouphierarchylevels: 0 - ranger.usersync.group.based.role.assignment.rules: '&ROLE_SYS_ADMIN:g:field-se' - ranger.usersync.group.memberattributename: "{{ auth_provider.ldap_attr_group_membership }}" - ranger.usersync.group.nameattribute: "{{ auth_provider.ldap_attr_group_name }}" - ranger.usersync.group.objectclass: group - ranger.usersync.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" -# ranger.usersync.group.searchfilter: "" -# ranger.usersync.ldap.user.searchfilter: "" - ranger.usersync.ldap.user.nameattribute: "{{ auth_provider.ldap_attr_user_name }}" - ranger.usersync.ldap.user.objectclass: person - ranger.usersync.ldap.user.searchbase: "{{ auth_provider.ldap_search_base.user }}" - ranger.usersync.source.impl.class: org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder - ranger.usersync.user.searchenabled: 'true' - -#obsolete in 7.1.6 ranger.usersync.ldap.searchBase: "{{ auth_provider.ldap_base_dn }}" -#obsolete in 7.1.6 ranger.usersync.ldap.user.groupnameattribute: "{{ auth_provider.ldap_attr_group_name }}" -#obsolete in 7.1.6 ranger.usersync.group.search.first.enabled: 'true' -#obsolete in 7.1.6 ranger.usersync.group.searchenabled: 'true' - INFRA_SOLR: - SERVICEWIDE: - #set false for 7.1.7 since it doesnt build cm_solr - enable_ranger_authorization: false - SOLR_SERVER: - solr_graceful_stop_timeout: 5 - SPARK_ON_YARN: - SPARK_YARN_HISTORY_SERVER: - ssl_enabled: true - history_server_spnego_enabled: false - YARN: - SERVICEWIDE: - hadoop_secure_web_ui: false - JOBHISTORY: - mr2_jobhistory_java_heapsize: 134217728 - NODEMANAGER: - node_manager_java_heapsize: 134217728 - yarn_nodemanager_resource_memory_mb: 2048 - yarn_nodemanager_resource_cpu_vcores: 2 - RESOURCEMANAGER: - resource_manager_java_heapsize: 134217728 - yarn_scheduler_maximum_allocation_mb: 2048 - yarn_scheduler_maximum_allocation_vcores: 2 - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - service_config_suppression_server_count_validator: true - SERVER: - zookeeper_server_java_heapsize: 134217728 - host_templates: - Masters: - ATLAS: [ATLAS_SERVER] - HBASE: [MASTER, REGIONSERVER, GATEWAY] - HDFS: [NAMENODE, SECONDARYNAMENODE, DATANODE] - HIVE: [HIVEMETASTORE, GATEWAY] - HIVE_ON_TEZ: [HIVESERVER2, GATEWAY] - INFRA_SOLR: [SOLR_SERVER] - KAFKA: [KAFKA_BROKER] - OZONE: [GATEWAY, OZONE_MANAGER, OZONE_RECON, STORAGE_CONTAINER_MANAGER, OZONE_DATANODE, S3_GATEWAY] - RANGER: [RANGER_ADMIN, RANGER_TAGSYNC, RANGER_USERSYNC] - SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER, GATEWAY] - TEZ: [GATEWAY] - YARN: [RESOURCEMANAGER, NODEMANAGER, JOBHISTORY, GATEWAY] - ZOOKEEPER: [SERVER] - - Workers: - HBASE: [REGIONSERVER, GATEWAY] - HDFS: [DATANODE] - HIVE: [GATEWAY] - HIVE_ON_TEZ: [GATEWAY] - OZONE: [OZONE_DATANODE, GATEWAY] - TEZ: [GATEWAY] - SPARK_ON_YARN: [GATEWAY] - YARN: [NODEMANAGER, GATEWAY] - ZOOKEEPER: [SERVER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - security: - tls: true - configs: - HOSTMONITOR: - firehose_heapsize: 1073741824 - firehose_non_java_memory_bytes: 2147483648 - SERVICEMONITOR: - firehose_heapsize: 2147483648 - firehose_non_java_memory_bytes: 6442450944 - role_config_suppression_firehose_service_monitor_non_java_memory_role_validator: true - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true - host_config_suppression_memory_overcommitted_validator: true diff --git a/private-cloud/ecs-on-aws/config.yml b/private-cloud/ecs-on-aws/config.yml deleted file mode 100644 index 40ed695..0000000 --- a/private-cloud/ecs-on-aws/config.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - -name_prefix: "labaw" # CHANGE THIS -owner_prefix: "pvc-ecs" -owner_email: "example@cloudera.com" -infra_region: "eu-west-1" -infra_type: "aws" # "aws", "static" -domain: "{{ owner_prefix }}.cldr.example" # The private, adhoc subdomain (name_prefix.owner_prefix.cldr.demo) -realm: "CLDR.EXAMPLE" # The Kerberos realm -common_password: "Example776" -admin_password: "Example776" # "Main" default password -deployment_tags: - owner: "{{ owner_prefix }}" - email: "{{ owner_email }}" - project: "PvC ECS Lab - {{ owner_prefix }}-{{ name_prefix }}" - enddate: "{{ ('%m%d%Y' | strftime((ansible_date_time.epoch | int) + (90 * 86400))) }}" - deployment: "{{ name_prefix }}" - deploy-tool: cloudera-deploy diff --git a/private-cloud/ecs-on-aws/definition.yml b/private-cloud/ecs-on-aws/definition.yml deleted file mode 100644 index 9a04533..0000000 --- a/private-cloud/ecs-on-aws/definition.yml +++ /dev/null @@ -1,245 +0,0 @@ ---- - -kerberos_activated: True -encryption_activated: True -# tls_activated: True -# autotls: False - -repositories: - # Offical CDH 7.1.9.0 - - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/ -cloudera_manager_version: 7.11.3 - -jdk_version: 11 - -# ECS details -ecs_options: - # Version 1.5.1 - repository: https://archive.cloudera.com/p/cdp-pvc-ds/1.5.1/ - parcel_repositories: - - https://archive.cloudera.com/p/cdp-pvc-ds/1.5.1/parcels/ - -# SSH -public_key_id: "{{ owner_prefix }}-{{ name_prefix }}" -public_key_file: "{{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') }}" -public_key_text: "{{ lookup('ansible.builtin.file', lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE')) }}" - -# Cloudera license -license_file: "{{ lookup('ansible.builtin.env', 'CDP_LICENSE_FILE') }}" - -# Terraform settings -terraform: - state: - storage: local - create_remote_storage: False - s3_region: "{{ infra_region }}" - s3_bucket: "{{ owner_prefix }}-{{ name_prefix }}-tf-state" - -# DNS -dns_provider: "freeipa" - -# Connect FreeIPA to Knox and Ranger -freeipa_activated: yes -# FreeIPA client install on cluster nodes is done during pre-setup -freeipa_enroll: no - -# FreeIPA realm settings -freeipa: - realm: "{{ realm }}" - -##### - -# Vars from legacy cloudera-Deploy -use_auto_repo_mirror: no -use_default_cluster_definition: no -use_download_mirror: no -preload_cm_parcel_repo: yes -teardown_everything: yes - -# Defer user and group creation to SSSD/FreeIPA registration -#Use when you dont need ansible to run "ca_server" -skip_user_group_init: no - -# Cloudera Manager details - -cloudera_manager_options: - CUSTOM_BANNER_HTML: "1.5.1 - PvC ECS Lab ({{ name_prefix }})" - SESSION_TIMEOUT: 43200 - PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560 - KRB_AUTH_ENABLE: "true" - -# License options (this is due to a hardcoded tmp directory on the target/manager node) -license_local_tmp_path: /tmp/cloudera_license.txt - -############### -### RDBMS ### -############### -database_type: postgresql -database_version: 12 -database_tls: true -database_default_password: "{{ common_password }}" - -############### - -## Red Hat FreeIPA -krb5_kdc_type: Red Hat IPA -krb5_realm: "{{ realm }}" -krb5_kdc_admin_user: "{{ freeipa.ipaadmin_user | default(lookup('env', 'IPA_USER', default='Undefined')) }}@{{ krb5_realm }}" -krb5_kdc_admin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" -krb5_enc_types: "aes256-cts aes128-cts" -# TODO: This needs to either be explicit or rendered from a host with DNS resolution -# Cannot rely on localhost to have DNS access, e.g. private networking on EC2 -#krb5_kdc_host: "{{ lookup('community.general.dig', '_ldap._tcp.' + domain +'./SRV', 'flat=0', wantlist=True, fail_on_error=True) | map(attribute='target') | first }}" -krb5_kdc_host: "{{ groups['freeipa'] | first | default('') }}" - -############################################ -### CM External Auth - FreeIPA as LDAP ### -############################################ - -# Settings for FreeIPA sidecar deployment -cloudera_manager_external_auth: - provider: freeipa - external_first: no - external_only: no - -auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" - -base_dn: "dc={{ (krb5_realm | lower).split('.') | join(',dc=') }}" -user_dn: "cn=users,cn=accounts,{{ base_dn }}" -group_dn: "cn=groups,cn=accounts,{{ base_dn }}" - -auth_providers: - freeipa: - ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}:636" - ldap_base_dn: "{{ base_dn }}" - ldap_bind_user_dn: "uid=admin,{{ user_dn }}" - ldap_bind_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" - ldap_search_base: - user: "{{ user_dn }}" - group: "{{ group_dn }}" - ldap_object_class: - user: "person" - group: "groupofnames" - ldap_search_filter: - user: uid={0} - member: (member={0}) - ldap_attr_user_name: "uid" - ldap_attr_group_name: "cn" - ldap_attr_group_membership: "member" - type: LDAP - -## TLS signing -skip_ipa_signing: no # This is the default -remote_ipa_server: "{{ groups['freeipa'] | first | default('') }}" - -tls_key_password: "{{ common_password }}" -tls_keystore_password: "{{ common_password }}" -tls_truststore_password: "{{ common_password }}" - -ca_server_attrs_root: - CN: 'SE-Root CA' -ca_server_attrs_intermediate: - CN: 'SE-Intermediate CA' - -# Infrastructure -infra: - aws: - region: "{{ infra_region }}" - enable_dns: no - public_subnets: # TODO: Implement better selection of AZs - - az: "{{ infra_region }}a" - private_subnets: - - az: "{{ infra_region }}b" - - az: "{{ infra_region }}c" - default_security_group: - ingress: - - cidr: [ "0.0.0.0/0" ] - from: 22 - to: 22 - protocol: TCP - desc: SSH (Ansible) - nodes: - # Sidecar FreeIPA for Kerberos and DNS - - hostname_prefix: "freeipa" - count: 1 - subnet_index: 0 # TODO: Implement better selection of AZs. this is the combined list of public + private - groups: - - freeipa - - jump_host - - nodes - instance_type: m5.large - # Manager, Master, DB, and CA - - hostname_prefix: "base" # hostname will be of form {{ name_prefix }}-{{ hostname_prefix }}- - count: 1 - groups: - - cloudera_manager - - cluster_masters - - cluster - - nodes - - db_server - - linux_nodes - - deployment - host_template: Masters - instance_type: m5.4xlarge - root_volume: - volume_size: 250 - subnet_index: 1 - tls: True - # Cluster Worker - - hostname_prefix: worker - count: 2 - groups: - - cluster_workers - - cluster - - nodes - - linux_nodes - - deployment - instance_type: c5.2xlarge - subnet_index: 1 - root_volume: - volume_size: 250 - host_template: Workers - tls: True - # ECS Master - - hostname_prefix: ecs-m - count: 1 - groups: - - cluster_ecs_masters - - cluster - - nodes - - ecs_nodes - - small_ecs_nodes - - linux_nodes - - deployment - instance_type: c5.2xlarge - subnet_index: 1 - root_volume: - volume_size: 250 - volumes: - - device: /dev/sdb - mount: /ecs - size: 250 - type: gp2 - host_template: ECSMaster - tls: True - # ECS Workers - - hostname_prefix: ecs-w - count: 3 - groups: - - cluster_ecs_workers - - cluster - - nodes - - ecs_nodes - - linux_nodes - - deployment - instance_type: m5.8xlarge - subnet_index: 2 - root_volume: - volume_size: 250 - volumes: - - device: /dev/sdb - mount: /ecs - size: 250 - type: gp2 - host_template: ECSWorker - tls: True diff --git a/private-cloud/ecs-on-aws/external_setup.yml b/private-cloud/ecs-on-aws/external_setup.yml deleted file mode 100644 index cdbe151..0000000 --- a/private-cloud/ecs-on-aws/external_setup.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- - -- name: Set up CDP Private Cloud external prerequisites - hosts: localhost - connection: local - gather_facts: yes - vars: - definition_path: "./" - tasks: - - name: Set of deployment variables from definition.yml - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always - -- name: Set up external prerequisites for CDP Private Cloud - ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_ext.yml - -# NOTE: Cluster hosts become unavilable after krb5 client setup -# This is a temporary workaround while we debug the source of the issue -- name: Reboot all cluster hosts - hosts: cluster - gather_facts: no - become: yes - tasks: - - name: Reboot all cluster hosts - ansible.builtin.reboot: \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/group_vars/all.yml b/private-cloud/ecs-on-aws/group_vars/all.yml deleted file mode 100644 index 19f3184..0000000 --- a/private-cloud/ecs-on-aws/group_vars/all.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# If needed, set the following variable to tell Ansible which key to use -# ansible_ssh_private_key_file: ~/.ssh/some-private-key diff --git a/private-cloud/ecs-on-aws/hostvars.j2 b/private-cloud/ecs-on-aws/hostvars.j2 deleted file mode 100644 index bf2ca91..0000000 --- a/private-cloud/ecs-on-aws/hostvars.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{# Collect and output individual host variables #} -{% macro host_variables(host) %} -{% set fields = [] %} -{% set _ = fields.append("ansible_user=" + host['ansible_user']) if 'ansible_user' in host %} -{% set _ = fields.append("host_template=" + host['host_template']) if 'host_template' in host %} -{% set _ = fields.append("label=" + host['label']) if 'label' in host %} -{% set _ = fields.append("tls=" + host['tls'] | string) if 'tls' in host %} -{{ host['inventory_hostname'] }} {{ fields | join(' ') }} -{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/instance_vars.j2 b/private-cloud/ecs-on-aws/instance_vars.j2 deleted file mode 100644 index 6e7063f..0000000 --- a/private-cloud/ecs-on-aws/instance_vars.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{# Define the metadata tags for the individual Openstack instances #} -{# Output should be TF map _entries_, not a map itself #} - -{% macro instance_tags(host) %} -{% set tags = {} %} -{% set _ = tags.update({ 'ansible_user': host.ansible_user }) if host.ansible_user is defined %} -{% set _ = tags.update({ 'host_template': host.host_template }) if host.host_template is defined %} -{% set _ = tags.update({ 'groups': host.groups | join(', ') }) if host.groups is defined %} -{% set _ = tags.update({ 'tls': host.tls | string }) if host.tls is defined %} -{% for k, v in tags.items() %} - {{ k }} = "{{ v }}"{{ "," if not loop.last else "" }} -{% endfor %} -{%- endmacro %} \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/internal_setup.yml b/private-cloud/ecs-on-aws/internal_setup.yml deleted file mode 100644 index 18b56ea..0000000 --- a/private-cloud/ecs-on-aws/internal_setup.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: Set up CDP Private Cloud internal prerequisites - hosts: localhost - connection: local - gather_facts: yes - vars: - definition_path: "./" - tasks: - - name: Set of deployment variables from definition.yml - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - when: init__completed is undefined - - - name: Prepare inventory for PvC Plays - ansible.builtin.include_role: - name: cloudera.exe.init_deployment - public: yes - tasks_from: prep_pvc.yml - tags: - - always - -- name: Init run tasks for all nodes - hosts: all - gather_facts: no - tasks: - - name: Group hosts by host template and TLS - ansible.builtin.include_role: - name: cloudera.cluster.deployment.groupby - - - name: Check connectivity to Inventory - ansible.builtin.wait_for_connection: - tags: - - always - -- name: Set up internal prerequisites for CDP Private Cloud - ansible.builtin.import_playbook: cloudera.exe.pvc_base_prereqs_int.yml diff --git a/private-cloud/ecs-on-aws/pre_setup.yml b/private-cloud/ecs-on-aws/pre_setup.yml deleted file mode 100644 index b4035c5..0000000 --- a/private-cloud/ecs-on-aws/pre_setup.yml +++ /dev/null @@ -1,231 +0,0 @@ ---- - -# Run variable checker - -# Provision the infrastructure at the cloud provider -- name: Provision infrastructure resources - hosts: localhost - connection: local - gather_facts: yes - module_defaults: - # Set your definition-specific inventory host variables here. - # NOTE: the 'node' variable is in scope and contains each entry coming from - # the Terraform 'nodes' output variable. - ansible.builtin.add_host: - host_template: "{{ node.metadata.host_template | default(omit) }}" - tls: "{{ node.metadata.tls | default(omit) }}" - # TODO Review the SSH args, these are overwritten - ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -C -o ControlMaster=auto -o ControlPersist=1200s -o BatchMode=yes' - tags: infra - tasks: - - name: Provision the infrastructure resources in the cloud provider - when: infra_type != 'static' - ansible.builtin.import_role: - name: cloudera.exe.provision - vars: - provision_provider: "{{ infra_type }}" - provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini" - provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" - provision_terraform_parallelism: "{{ terraform.parallelism | default(omit) }}" - provision_state_storage: "{{ terraform.state.storage }}" - provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" - provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" - provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" - provision_name_prefix: "{{ name_prefix }}" - provision_domain_suffix: "{{ domain }}" - provision_ssh_keypair_label: "{{ ssh_keypair.label | default(public_key_id) }}" - provision_ssh_keypair_public_key: "{{ ssh_keypair.public_key | default(public_key_text) }}" - provision_owner_email: "{{ owner_email }}" - provision_openstack_network_name: "{{ infra[infra_type].network_name | default(omit) }}" - provision_openstack_default_image_name: "{{ infra[infra_type].default_image_name | default(omit) }}" - provision_openstack_default_availability_zone: "{{ infra[infra_type].default_availability_zone | default(omit) }}" - provision_aws_ec2_region: "{{ infra[infra_type].region | default(omit) }}" - provision_aws_ec2_vpc_enable_dns_support: "{{ infra[infra_type].enable_dns | default(omit) }}" - provision_aws_ec2_vpc_enable_dns_hostnames: "{{ infra[infra_type].enable_dns | default(omit) }}" - provision_aws_ec2_public_subnets: "{{ infra[infra_type].public_subnets | default(omit) }}" - provision_aws_ec2_private_subnets: "{{ infra[infra_type].private_subnets | default(omit) }}" - provision_aws_ec2_default_security_group_ingress: "{{ infra[infra_type].default_security_group.ingress | default(omit) }}" - provision_instances: "{{ infra[infra_type].nodes | default([]) }}" - provision_tags: "{{ deployment_tags | default(omit) }}" - -# Confirm the availablity of all nodes -- name: Ensure node readiness - hosts: nodes - gather_facts: no - tasks: - - name: Ensure the node is reachable - ansible.builtin.wait_for_connection: - timeout: 60 - -# Prepare and mount any attached volumes -- name: Prepare and mount storage volumes - hosts: nodes - gather_facts: no - become: True - tasks: - - name: Prepare storage volumes - when: infra_type != "static" and storage_volumes | length > 0 - ansible.builtin.import_role: - name: cloudera.exe.mount - vars: - mount_volumes: "{{ storage_volumes }}" - mount_provider: "{{ infra_type }}" - -# Prepare the FreeIPA sidecar services -- name: Provision FreeIPA services - hosts: freeipa - gather_facts: yes - become: yes - vars: - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" - # If not set specifically: if AWS, then VPC CIDR + 2 (see https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) - resolv_nameservers: "{{ upstream_nameservers | default(default_dns[infra_type]) }}" - default_dns: - aws: "{{ [ hostvars.localhost.provision.vpc.cidr | ansible.utils.ipmath(2) ] }}" - module_defaults: - freeipa.ansible_freeipa.ipadnszone: - ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" - freeipa.ansible_freeipa.ipadnsrecord: - ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" - tasks: - - name: Set up the FreeIPA server - ansible.builtin.import_role: - name: cloudera.exe.freeipa_server - vars: - ipaserver_hostname: "{{ inventory_hostname }}" - ipaserver_realm: "{{ freeipa.realm }}" - ipaserver_domain: "{{ name_prefix }}.{{ domain }}" - ipaserver_no_host_dns: yes - ipaserver_setup_firewalld: no - ipaserver_setup_dns: "{{ enable_dns }}" - ipaserver_resolv_nameservers: "{{ resolv_nameservers }}" - ipaserver_auto_reverse: "{{ enable_dns | ternary(True, omit) }}" - ipaserver_no_forwarders: "{{ enable_dns | ternary(True, omit) }}" - ipaserver_forward_policy: "{{ enable_dns | ternary('only', omit) }}" - ipaserver_recursion_acl_cidr: "{{ enable_dns | ternary(hostvars.localhost.provision.vpc.cidr, omit) }}" - ipaserver_copy_csr_to_controller: yes - ipaclient_mkhomedir: yes - ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" # TODO Add test for these parameters - error in role is opaque - ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" - - - name: Configure provisioned FreeIPA global DNS - when: enable_dns - block: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in provisioned FreeIPA service - freeipa.ansible_freeipa.ipadnszone: - zone_name: "{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - forward_policy: none - - - name: Create reverse DNS zone for '{{ hostvars.localhost.provision.vpc.cidr }}' - freeipa.ansible_freeipa.ipadnszone: - name_from_ip: "{{ hostvars.localhost.provision.vpc.cidr }}" - dynamic_update: yes - allow_sync_ptr: yes - - - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' - when: "'cluster_ecs_masters' in groups" - block: - - name: Ensure DNS zone for ECS 'apps' - freeipa.ansible_freeipa.ipadnszone: - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - forward_policy: none - - - name: Ensure DNS wildcard records for ECS 'apps' and domain - freeipa.ansible_freeipa.ipadnsrecord: - name: "*" - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - record_type: "A" - record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" - create_reverse: no - - - name: Resolve provisioned FreeIPA DNS server addresses - ansible.builtin.set_fact: - dns_server_ips: "{{ dns_server_ips | default([]) | union([ ansible_default_ipv4['address'] ]) }}" - delegate_to: localhost - delegate_facts: true - -# Update existing FreeIPA DNS services -- name: Configure existing FreeIPA global DNS - hosts: localhost - connection: local - gather_facts: no - vars: - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" - dns_server: "{{ freeipa.server | default(lookup('ansible.builtin.env', 'IPA_HOST', default='Undefined')) }}" - module_defaults: - community.general.ipa_dnszone: - ipa_host: "{{ dns_server }}" - ipa_user: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default='Undefined')) }}" - ipa_pass: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" - tasks: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' in existing FreeIPA service - when: enable_dns and dns_server is defined and 'freeipa' not in groups - block: - - name: Create DNS zone for '{{ name_prefix }}.{{ domain }}' - community.general.ipa_dnszone: - validate_certs: no - dynamicupdate: yes - allowsyncptr: yes - zone_name: "{{ name_prefix }}.{{ domain }}" - state: present - - - name: Create reverse DNS zone for '{{ infra_cidr }}' - community.general.ipa_dnszone: - validate_certs: no - dynamicupdate: yes - allowsyncptr: yes - zone_name: "{{ infra_cidr | ansible.utils.ipaddr('revdns') }}" - state: present - - - name: Update DNS for ECS 'apps.{{ name_prefix }}.{{ domain }}' - when: "'cluster_ecs_masters' in groups" - block: - - name: Ensure DNS zone for ECS 'apps' - community.general.ipa_dnszone: - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - dynamic_update: yes - allow_sync_ptr: yes - validate_certs: no - - - name: Ensure DNS wildcard records for ECS 'apps' and domain - community.general.ipa_dnszone: - name: "*" - zone_name: "apps.{{ name_prefix }}.{{ domain }}" - record_type: "A" - record_value: "{{ hostvars[groups['cluster_ecs_masters'][0]]['ansible_host'] }}" - validate_certs: no - - - name: Resolve existing FreeIPA DNS server addresses - ansible.builtin.set_fact: - dns_hosts: "{{ [ dns_server ] }}" - dns_server_ips: "{{ [ lookup('community.general.dig', dns_server ) ] }}" - -# Enroll deployment hosts with FreeIPA Kerberos and possibly with DNS -- name: Register hosts with FreeIPA - hosts: nodes:!freeipa - gather_facts: yes - become: True - tasks: - - name: Register host with FreeIPA services - when: krb5_kdc_type == "Red Hat IPA" - ansible.builtin.import_role: - name: cloudera.exe.freeipa_client - vars: - ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}" - ipaserver_realm: "{{ freeipa.realm }}" - ipa_hosts: "{{ groups['freeipa'] | default(hostvars.localhost.dns_hosts) }}" - ipa_server_ips: "{{ hostvars.localhost.dns_server_ips }}" - ipaadmin_password: "{{ freeipa.ipaadmin_password | default(lookup('ansible.builtin.env', 'IPA_PASSWORD', default='Undefined')) }}" - ipaadmin_principal: "{{ freeipa.ipaadmin_user | default(lookup('ansible.builtin.env', 'IPA_USER', default=omit)) }}" - enable_dns: "{{ dns_provider | lower == 'freeipa' }}" - -# Prepare SME Lab resources -- name: Prepare TLS and supporting services resources - import_playbook: pre_setup_resources.yml - tags: - - prereq - - infra \ No newline at end of file diff --git a/private-cloud/ecs-on-aws/pre_setup_resources.yml b/private-cloud/ecs-on-aws/pre_setup_resources.yml deleted file mode 100644 index e4270be..0000000 --- a/private-cloud/ecs-on-aws/pre_setup_resources.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - # Playbook to setup additional resources for PvC Deployment - -- name: Ensure additional requirements are available - hosts: nodes - gather_facts: no - become: yes - tasks: - - - name: Install required packages - ansible.builtin.package: - name: - - tree - - jq - - zip - - python38 - state: present - - - name: Install required packages - ansible.builtin.pip: - name: psycopg2-binary==2.9.5 - executable: pip3.8 - state: present - diff --git a/private-cloud/ecs-on-aws/pre_teardown.yml b/private-cloud/ecs-on-aws/pre_teardown.yml deleted file mode 100644 index 7ceedf0..0000000 --- a/private-cloud/ecs-on-aws/pre_teardown.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -- name: Deprovision the infrastructure resources - hosts: localhost - connection: local - gather_facts: no - tags: - - teardown - - provision - tasks: - - name: Deprovision the infrastructure resources in the cloud provider - when: groups['all'] | length > 0 - ansible.builtin.import_role: - name: cloudera.exe.provision - vars: - provision_state: absent - provision_provider: "{{ infra_type }}" - provision_inventory_file: inventory_static.ini - provision_directory: "tf_deployment_{{ name_prefix }}_{{ infra_type }}" - provision_state_storage: "{{ terraform.state.storage }}" - provision_remote_storage_s3_region: "{{ terraform.state.s3_region }}" - provision_remote_storage_s3_bucket: "{{ terraform.state.s3_bucket }}" - provision_create_remote_storage: "{{ terraform.state.create_remote_storage }}" - provision_name_prefix: "{{ name_prefix }}" diff --git a/private-cloud/ecs-on-aws/validate_dns_lookups.yml b/private-cloud/ecs-on-aws/validate_dns_lookups.yml deleted file mode 100644 index 236169e..0000000 --- a/private-cloud/ecs-on-aws/validate_dns_lookups.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: Check that all hosts in inventory are reachable - hosts: all - gather_facts: no - tasks: - - name: Ping each host from controller - ping: - - -- name: Check Forward and Reverse DNS lookups - hosts: localhost - connection: local - gather_facts: no - tasks: - - - name: Print the list of hosts - debug: - var: groups['all'] - - - name: Forward lookup - ansible.builtin.debug: - msg: - - "Forward lookup DNS for {{ item }} is {{ lookup('community.general.dig', item , qtype='A') }}" - loop: "{{ groups['all'] }}" - - # TODO Register forward lookup IP and do reverse lookup - - From f1aa40202f06d8bc5f6180f4b7bad983587f5634 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 11 Dec 2023 14:25:14 -0500 Subject: [PATCH 42/45] Update LDAP references for Ranger and Atlas Signed-off-by: Webster Mudge --- private-cloud/base/aws-iaas/cluster.yml | 14 +++++++------- private-cloud/base/aws-iaas/definition.yml | 11 ++++++----- .../base/aws-iaas/tf_proxied_cluster/main.tf | 8 -------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/private-cloud/base/aws-iaas/cluster.yml b/private-cloud/base/aws-iaas/cluster.yml index b3f7a90..70322cb 100644 --- a/private-cloud/base/aws-iaas/cluster.yml +++ b/private-cloud/base/aws-iaas/cluster.yml @@ -42,7 +42,7 @@ clusters: atlas_authentication_method_ldap_groupSearchBase: "{{ auth_provider.ldap_search_base.group }}" atlas_authentication_method_ldap_groupSearchFilter: "{{ auth_provider.ldap_search_filter.member }}" atlas_authentication_method_ldap_groupRoleAttribute: "{{ auth_provider.ldap_attr_group_name }}" - atlas_authentication_method_ldap_type: "ldap" + atlas_authentication_method_ldap_type: "{{ auth_provider.type | lower }}" atlas_authentication_method_ldap_default_role: "ROLE_ADMIN" HBASE: SERVICEWIDE: @@ -126,14 +126,14 @@ clusters: rangerusersync_user_password: "{{ common_password }}" RANGER_ADMIN: ranger_admin_max_heap_size: 1024 - ranger_authentication_method: "LDAP" + ranger_authentication_method: "{{ auth_provider.type | upper }}" ranger.ldap.url: "{{ auth_provider.ldap_url }}" ranger.ldap.bind.dn: "{{ auth_provider.ldap_bind_user_dn }}" ranger_ldap_bind_password: "{{ auth_provider.ldap_bind_password }}" ranger.ldap.user.dnpattern: "uid={0},{{ auth_provider.ldap_search_base.user }}" ranger.ldap.base.dn: "{{ auth_provider.ldap_base_dn }}" - ranger.ldap.user.searchfilter: "(&(sAMAccountName={0})(objectClass=person))" - ranger.ldap.group.searchfilter: "(&(member={0})(objectClass=group))" + ranger.ldap.user.searchfilter: "{{ auth_provider.ldap_search_filter.user }}" + ranger.ldap.group.searchfilter: "{{ auth_provider.ldap_search_filter.member }}" ranger.ldap.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" ranger.ldap.group.roleattribute: "{{ auth_provider.ldap_attr_group_name }}" ranger.ldap.referral: "follow" @@ -148,15 +148,15 @@ clusters: ranger.usersync.ldap.username.caseconversion: lower ranger.usersync.ldap.groupname.caseconversion: lower ranger.usersync.ldap.grouphierarchylevels: 0 - ranger.usersync.group.based.role.assignment.rules: '&ROLE_SYS_ADMIN:g:field-se' + # ranger.usersync.group.based.role.assignment.rules: '&ROLE_SYS_ADMIN:g:field-se' ranger.usersync.group.memberattributename: "{{ auth_provider.ldap_attr_group_membership }}" ranger.usersync.group.nameattribute: "{{ auth_provider.ldap_attr_group_name }}" - ranger.usersync.group.objectclass: group + ranger.usersync.group.objectclass: "{{ auth_provider.ldap_object_class.group }}" # group ranger.usersync.group.searchbase: "{{ auth_provider.ldap_search_base.group }}" # ranger.usersync.group.searchfilter: "" # ranger.usersync.ldap.user.searchfilter: "" ranger.usersync.ldap.user.nameattribute: "{{ auth_provider.ldap_attr_user_name }}" - ranger.usersync.ldap.user.objectclass: person + ranger.usersync.ldap.user.objectclass: "{{ auth_provider.ldap_object_class.user }}" # person ranger.usersync.ldap.user.searchbase: "{{ auth_provider.ldap_search_base.user }}" ranger.usersync.source.impl.class: org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder ranger.usersync.user.searchenabled: 'true' diff --git a/private-cloud/base/aws-iaas/definition.yml b/private-cloud/base/aws-iaas/definition.yml index e652175..177be9b 100644 --- a/private-cloud/base/aws-iaas/definition.yml +++ b/private-cloud/base/aws-iaas/definition.yml @@ -103,6 +103,7 @@ cloudera_manager_external_auth: external_first: no external_only: no +# Set in cloudera_manager.external_auth.tasks.main:26 auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" base_dn: "dc={{ freeipa.realm.split('.') | map('lower') | join(',dc=') }}" @@ -111,7 +112,7 @@ group_dn: "cn=groups,cn=accounts,{{ base_dn }}" auth_providers: freeipa: - ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}:636" + ldap_url: "ldaps://{{ groups['freeipa'] | first | default('') }}" ldap_base_dn: "{{ base_dn }}" ldap_bind_user_dn: "uid=admin,{{ user_dn }}" ldap_bind_password: "{{ freeipa.ipaadmin_password }}" @@ -121,11 +122,11 @@ auth_providers: ldap_object_class: user: "person" group: "posixgroup" - ldap_search_filter: - user: uid - member: member + ldap_search_filter: # Used for mapping Services (Atlas, Ranger, etc.) to LDAP + user: "(&(sAMAccountName={0})(objectClass=person))" + member: "(&(member={0})(objectClass=group))" # group: (&(member={0})(objectclass=posixgroup)(!(cn=admins))) - ldap_attribute: + ldap_attribute: # Used for mapping CM to LDAP user: uid # Set to (uid={0}) member: member # defaults to (member={0}) ldap_attr_user_name: "uid" diff --git a/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf b/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf index 723c104..b7b5cf2 100644 --- a/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf +++ b/private-cloud/base/aws-iaas/tf_proxied_cluster/main.tf @@ -263,14 +263,6 @@ resource "ansible_group" "deployment" { } } -# resource "ansible_group" "nodes" { -# name = "nodes" -# children = [ -# ansible_group.deployment.name, -# ansible_group.bastion.name -# ] -# } - resource "ansible_host" "masters" { for_each = { for idx, host in module.masters.hosts : idx => host } From e1e37ecc7c82544449383b06f932789f0eeb440e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 11 Dec 2023 14:27:46 -0500 Subject: [PATCH 43/45] Add summary playbook for post-installation use Signed-off-by: Webster Mudge --- private-cloud/base/aws-iaas/.gitignore | 1 + private-cloud/base/aws-iaas/summary.yml | 7 +- .../aws-iaas/templates/deployment.html.j2 | 150 ++++++++++++++++++ .../base/aws-iaas/templates/deployment.md.j2 | 8 +- 4 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 private-cloud/base/aws-iaas/templates/deployment.html.j2 diff --git a/private-cloud/base/aws-iaas/.gitignore b/private-cloud/base/aws-iaas/.gitignore index 1040cfb..e6cda98 100644 --- a/private-cloud/base/aws-iaas/.gitignore +++ b/private-cloud/base/aws-iaas/.gitignore @@ -40,5 +40,6 @@ inventory_static* config.yml # Per-deployment artifacts +*-DEPLOYMENT.html *-DEPLOYMENT.md *.ca.crt diff --git a/private-cloud/base/aws-iaas/summary.yml b/private-cloud/base/aws-iaas/summary.yml index ba1c8d0..4cc24f1 100644 --- a/private-cloud/base/aws-iaas/summary.yml +++ b/private-cloud/base/aws-iaas/summary.yml @@ -17,10 +17,15 @@ connection: local gather_facts: no tasks: - - name: Create the deployment summary document + - name: Create the deployment summary Markdown document ansible.builtin.template: src: "deployment.md.j2" dest: "{{ name_prefix | upper }}-DEPLOYMENT.md" + + - name: Create the deployment summary HTML document + ansible.builtin.template: + src: "deployment.html.j2" + dest: "{{ name_prefix | upper }}-DEPLOYMENT.html" - name: Retrieve the FreeIPA CA certificate hosts: freeipa diff --git a/private-cloud/base/aws-iaas/templates/deployment.html.j2 b/private-cloud/base/aws-iaas/templates/deployment.html.j2 new file mode 100644 index 0000000..4aacb09 --- /dev/null +++ b/private-cloud/base/aws-iaas/templates/deployment.html.j2 @@ -0,0 +1,150 @@ + + + + + + + + + + + {{ name_prefix}} - PVC Base Deployment + + + +
+
+

{{ name_prefix}} - PVC Base Deployment

+

A CDP Private Cloud Base cluster running on AWS IaaS.

+
+ + + + + + + + + + + + + + + + + +
Prefix{{ name_prefix }}
Domain{{ domain }}
Realm + {{ realm }}
SSH + {{ lookup('ansible.builtin.env', 'SSH_PUBLIC_KEY_FILE') | basename }}
+

Deployment Access

+

Enable the SSH tunnel to the deployment:

+ 
+  ssh -D 8157 -qCN {{ groups['jump_host'] | map('extract', hostvars, 'ansible_user') | first }}@{{ groups['jump_host'] | map('extract', hostvars, 'ansible_host') | first }}
+
+

Use a SOCKS5 proxy switcher in your browser to access the deployment endpoints. (An example is the + SwitchyOmega extension for Chrome-based browsers.)

+

In the proxy configuration, set Protocol to SOCKS5, Server to localhost, and + Port to 8157. Ensure the SOCKS5 proxy is active when clicking on the CDP Private Cloud Base UIs and + endpoints that you wish to access.

+

SSL Certificate Installation

+

Any SSL-enabled endpoint within the cluster will produce a SSL warning for the self-signed certificate issued by + the deployment's local FreeIPA server. If you wish to avoid this warning, install the CA certificate, {{ + groups['freeipa'] | first }}.ca.crt, for your browser and/or system.

+

Endpoints

+ + + + + + + + + + + +
Cloudera Managerhttps://{{ + groups['cloudera_manager'] | first }}:7183
FreeIPAhttps://{{ groups['freeipa'] | first + }}
+

Hosts

+ + + + + + + + + + {% for host in groups['all'] | sort | map('extract', hostvars) | list %} + + + + + + {% endfor %} + +
HostFQDNGroups
{{ host['inventory_hostname_short'] }}{{ host['inventory_hostname'] }}{{ host['group_names'] | join(', ') }}
+
+ + + \ No newline at end of file diff --git a/private-cloud/base/aws-iaas/templates/deployment.md.j2 b/private-cloud/base/aws-iaas/templates/deployment.md.j2 index da84903..33401bf 100644 --- a/private-cloud/base/aws-iaas/templates/deployment.md.j2 +++ b/private-cloud/base/aws-iaas/templates/deployment.md.j2 @@ -1,5 +1,7 @@ # {{ name_prefix}} - PVC Base Deployment +> A CDP Private Cloud Base cluster running on AWS IaaS. + * **Prefix:** {{ name_prefix }} * **Domain:** {{ domain }} * **Realm:** {{ realm }} @@ -34,8 +36,8 @@ for your browser and/or system. ## Hosts -|Host|FQDN| -|---|---| +|Host|FQDN|Groups| +|---|---|---| {% for host in groups['all'] | sort | map('extract', hostvars) | list %} -| {{ host['inventory_hostname_short'] }} | {{ host['inventory_hostname'] }} | +| {{ host['inventory_hostname_short'] }} | {{ host['inventory_hostname'] }} | {{ host['group_names'] | join(', ') }} {% endfor %} From d34c3a9e11751dd80f8d02624468b23077b0bf26 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Sun, 17 Dec 2023 23:30:42 -0500 Subject: [PATCH 44/45] Update CA Subject for FreeIPA server Signed-off-by: Webster Mudge --- private-cloud/base/aws-iaas/pre_setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/private-cloud/base/aws-iaas/pre_setup.yml b/private-cloud/base/aws-iaas/pre_setup.yml index 8c06186..5b5b914 100644 --- a/private-cloud/base/aws-iaas/pre_setup.yml +++ b/private-cloud/base/aws-iaas/pre_setup.yml @@ -93,6 +93,7 @@ ipaserver_forward_policy: only ipaserver_recursion_acl_cidr: "{{ vpc_cidr }}" ipaserver_copy_csr_to_controller: yes + ipaserver_ca_subject: "CN=CLDR-{{ prefix_name }}-RootCA,O={{ freeipa.realm }}" ipaclient_mkhomedir: yes # TODO Add test for these parameters - error in role is opaque ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}" From 80dcd0fe8e87c5eda396312e2d2ce40224882d47 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 18 Dec 2023 15:33:22 -0500 Subject: [PATCH 45/45] Fix errant prefix variable Signed-off-by: Webster Mudge --- private-cloud/base/aws-iaas/pre_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private-cloud/base/aws-iaas/pre_setup.yml b/private-cloud/base/aws-iaas/pre_setup.yml index 5b5b914..e57488c 100644 --- a/private-cloud/base/aws-iaas/pre_setup.yml +++ b/private-cloud/base/aws-iaas/pre_setup.yml @@ -93,7 +93,7 @@ ipaserver_forward_policy: only ipaserver_recursion_acl_cidr: "{{ vpc_cidr }}" ipaserver_copy_csr_to_controller: yes - ipaserver_ca_subject: "CN=CLDR-{{ prefix_name }}-RootCA,O={{ freeipa.realm }}" + ipaserver_ca_subject: "CN=CLDR-{{ name_prefix }}-RootCA,O={{ freeipa.realm }}" ipaclient_mkhomedir: yes # TODO Add test for these parameters - error in role is opaque ipadm_password: "{{ freeipa.ipaadmin_password | default(lookup('env', 'IPA_PASSWORD', default='Undefined')) }}"