From d7d367e09ae46241c3530525995d8ad667a24c89 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:43 -0400 Subject: [PATCH 01/72] Add Accumulo prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_accumulo/README.md | 49 +++ roles/prereq_accumulo/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 32 ++ roles/prereq_accumulo/tasks/main.yml | 39 ++ roles/prereq_accumulo/vars/main.yml | 19 + 11 files changed, 785 insertions(+) create mode 100644 roles/prereq_accumulo/README.md create mode 100644 roles/prereq_accumulo/meta/argument_specs.yml create mode 100644 roles/prereq_accumulo/molecule/default/converge.yml create mode 100644 roles/prereq_accumulo/molecule/default/create.yml create mode 100644 roles/prereq_accumulo/molecule/default/destroy.yml create mode 100644 roles/prereq_accumulo/molecule/default/molecule.yml create mode 100644 roles/prereq_accumulo/molecule/default/prepare.yml create mode 100644 roles/prereq_accumulo/molecule/default/requirements.yml create mode 100644 roles/prereq_accumulo/molecule/default/verify.yml create mode 100644 roles/prereq_accumulo/tasks/main.yml create mode 100644 roles/prereq_accumulo/vars/main.yml diff --git a/roles/prereq_accumulo/README.md b/roles/prereq_accumulo/README.md new file mode 100644 index 00000000..1daeffe3 --- /dev/null +++ b/roles/prereq_accumulo/README.md @@ -0,0 +1,49 @@ +# prereq_accumulo + +Set up for Accumulo + +This role prepares a host for Accumulo usage by creating a dedicated system user and group named `accumulo`. This user is essential for running Accumulo processes with appropriate permissions and isolation. + +The role will: +- Create the `accumulo` system user and group. +- Configure home directories and other necessary local paths for the `accumulo` user, if required. +- Ensure appropriate permissions are set for files and directories related to Accumulo. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: accumulo_nodes + tasks: + - name: Set up the accumulo user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_accumulo + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_accumulo/meta/argument_specs.yml b/roles/prereq_accumulo/meta/argument_specs.yml new file mode 100644 index 00000000..b6f5e983 --- /dev/null +++ b/roles/prereq_accumulo/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Accumulo + description: | + Set up for Accumulo usage, notably, create the local C(accumulo) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_accumulo/molecule/default/converge.yml b/roles/prereq_accumulo/molecule/default/converge.yml new file mode 100644 index 00000000..9d6f61ee --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Accumulo + ansible.builtin.import_role: + name: prereq_accumulo diff --git a/roles/prereq_accumulo/molecule/default/create.yml b/roles/prereq_accumulo/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_accumulo/molecule/default/destroy.yml b/roles/prereq_accumulo/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_accumulo/molecule/default/molecule.yml b/roles/prereq_accumulo/molecule/default/molecule.yml new file mode 100644 index 00000000..118f2a88 --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_accumulo-rhel9-4 + Project: Molecule testing for prereq_accumulo +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_accumulo/molecule/default/prepare.yml b/roles/prereq_accumulo/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_accumulo/molecule/default/requirements.yml b/roles/prereq_accumulo/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_accumulo/molecule/default/verify.yml b/roles/prereq_accumulo/molecule/default/verify.yml new file mode 100644 index 00000000..0683df6a --- /dev/null +++ b/roles/prereq_accumulo/molecule/default/verify.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for accumulo user + ansible.builtin.command: grep accumulo /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ accumulo_local_accounts }}" diff --git a/roles/prereq_accumulo/tasks/main.yml b/roles/prereq_accumulo/tasks/main.yml new file mode 100644 index 00000000..8e8e32a2 --- /dev/null +++ b/roles/prereq_accumulo/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ accumulo_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ accumulo_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_accumulo/vars/main.yml b/roles/prereq_accumulo/vars/main.yml new file mode 100644 index 00000000..bf8ae0b3 --- /dev/null +++ b/roles/prereq_accumulo/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +accumulo_local_accounts: + - user: accumulo + home: /var/lib/accumulo + comment: Accumulo From 3f68d3714dbcb7cb5ad4c06039c199a8fc5e431c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:44 -0400 Subject: [PATCH 02/72] Add ActivityMonitor prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_activitymonitor/README.md | 79 ++++ .../prereq_activitymonitor/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 32 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_activitymonitor/tasks/main.yml | 24 ++ 11 files changed, 852 insertions(+) create mode 100644 roles/prereq_activitymonitor/README.md create mode 100644 roles/prereq_activitymonitor/defaults/main.yml create mode 100644 roles/prereq_activitymonitor/meta/argument_specs.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/converge.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/create.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/destroy.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/molecule.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/prepare.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/requirements.yml create mode 100644 roles/prereq_activitymonitor/molecule/default/verify.yml create mode 100644 roles/prereq_activitymonitor/tasks/main.yml diff --git a/roles/prereq_activitymonitor/README.md b/roles/prereq_activitymonitor/README.md new file mode 100644 index 00000000..e98dcfb7 --- /dev/null +++ b/roles/prereq_activitymonitor/README.md @@ -0,0 +1,79 @@ +# prereq_activitymonitor + +Set up database and user accounts for Activity Monitor + +This role automates the setup of a database and its associated user accounts specifically for Cloudera's Activity Monitor service. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `activity_monitor_database`. +- Create a new database user specified by `activity_monitor_username` with the password from `activity_monitor_password`. +- Grant ownership and all necessary privileges to the `activity_monitor_username` for the new database. +- Ensure the database is configured correctly for Activity Monitor operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `activity_monitor_username` | `str` | `False` | `amon` | The username for the Activity Monitor database user. This user will also be the owner of the database. | +| `activity_monitor_password` | `str` | `False` | `amon` | The password for the Activity Monitor database user. It is highly recommended to override this default in production. | +| `activity_monitor_database` | `str` | `False` | `amon` | The name of the database to be created for Activity Monitor. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Activity Monitor database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_activitymonitor + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Activity Monitor database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_activitymonitor + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + activity_monitor_username: "my_amon_user" + activity_monitor_password: "a_strong_amon_password" + activity_monitor_database: "my_amon_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_activitymonitor/defaults/main.yml b/roles/prereq_activitymonitor/defaults/main.yml new file mode 100644 index 00000000..49e1b457 --- /dev/null +++ b/roles/prereq_activitymonitor/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +activity_monitor_username: amon +activity_monitor_password: amon +activity_monitor_database: amon diff --git a/roles/prereq_activitymonitor/meta/argument_specs.yml b/roles/prereq_activitymonitor/meta/argument_specs.yml new file mode 100644 index 00000000..5a6d4556 --- /dev/null +++ b/roles/prereq_activitymonitor/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Activity Monitor + description: + - Set up the Activity Monitor database and its associated user accounts, ensuring proper configuration for Activity Monitor operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `activity_monitor_username`, + `activity_monitor_password`, and `activity_monitor_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + activity_monitor_username: + description: The username for the Activity Monitor database user and owner of the database. + type: str + required: false + default: amon + activity_monitor_password: + description: The password for the Activity Monitor database user. + type: str + required: false + default: amon + activity_monitor_database: + description: The name of the database to be created for Activity Monitor. + type: str + required: false + default: amon diff --git a/roles/prereq_activitymonitor/molecule/default/converge.yml b/roles/prereq_activitymonitor/molecule/default/converge.yml new file mode 100644 index 00000000..3d89838b --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Activity Monitor database and configure its associated user account. + ansible.builtin.import_role: + name: prereq_activitymonitor diff --git a/roles/prereq_activitymonitor/molecule/default/create.yml b/roles/prereq_activitymonitor/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_activitymonitor/molecule/default/destroy.yml b/roles/prereq_activitymonitor/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_activitymonitor/molecule/default/molecule.yml b/roles/prereq_activitymonitor/molecule/default/molecule.yml new file mode 100644 index 00000000..90599c6e --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_activitymonitor-rhel9-4 + Project: Molecule testing for prereq_activitymonitor +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_activitymonitor/molecule/default/prepare.yml b/roles/prereq_activitymonitor/molecule/default/prepare.yml new file mode 100644 index 00000000..2dcd1235 --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/prepare.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres + when: database_type == 'postgresql' diff --git a/roles/prereq_activitymonitor/molecule/default/requirements.yml b/roles/prereq_activitymonitor/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_activitymonitor/molecule/default/verify.yml b/roles/prereq_activitymonitor/molecule/default/verify.yml new file mode 100644 index 00000000..e91a368c --- /dev/null +++ b/roles/prereq_activitymonitor/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'amon';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database amon does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'amon';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User amon does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_activitymonitor/tasks/main.yml b/roles/prereq_activitymonitor/tasks/main.yml new file mode 100644 index 00000000..64713967 --- /dev/null +++ b/roles/prereq_activitymonitor/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ activity_monitor_details }}" + activity_monitor_details: + - user: "{{ activity_monitor_username }}" + password: "{{ activity_monitor_password }}" + db: "{{ activity_monitor_database }}" + no_log: true From 72733298b07fb559971e88b035da1bf69ffe6c3c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:44 -0400 Subject: [PATCH 03/72] Add Atlas prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_atlas/README.md | 52 +++ roles/prereq_atlas/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_atlas/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_atlas/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ .../prereq_atlas/molecule/default/verify.yml | 38 ++ roles/prereq_atlas/tasks/main.yml | 39 ++ roles/prereq_atlas/vars/main.yml | 20 ++ 10 files changed, 750 insertions(+) create mode 100644 roles/prereq_atlas/README.md create mode 100644 roles/prereq_atlas/meta/argument_specs.yml create mode 100644 roles/prereq_atlas/molecule/default/converge.yml create mode 100644 roles/prereq_atlas/molecule/default/create.yml create mode 100644 roles/prereq_atlas/molecule/default/destroy.yml create mode 100644 roles/prereq_atlas/molecule/default/molecule.yml create mode 100644 roles/prereq_atlas/molecule/default/requirements.yml create mode 100644 roles/prereq_atlas/molecule/default/verify.yml create mode 100644 roles/prereq_atlas/tasks/main.yml create mode 100644 roles/prereq_atlas/vars/main.yml diff --git a/roles/prereq_atlas/README.md b/roles/prereq_atlas/README.md new file mode 100644 index 00000000..fdbdc8d0 --- /dev/null +++ b/roles/prereq_atlas/README.md @@ -0,0 +1,52 @@ +# prereq_atlas + +Set up for Atlas + +This role prepares a host for Atlas usage by creating a dedicated system user and group named `atlas`. This user is essential for running Atlas processes with appropriate permissions and isolation within a Hadoop environment. + +The role will: +- Create the `atlas` system user and group. +- Configure home directories and other necessary local paths for the `atlas` user, if required. +- Ensure appropriate permissions are set for files and directories related to Atlas. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: atlas_nodes + tasks: + - name: Set up the atlas user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_atlas +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_atlas/meta/argument_specs.yml b/roles/prereq_atlas/meta/argument_specs.yml new file mode 100644 index 00000000..ad33beaf --- /dev/null +++ b/roles/prereq_atlas/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Atlas + description: | + Set up for Hadoop usage, notably, create the local C(atlas) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_atlas/molecule/default/converge.yml b/roles/prereq_atlas/molecule/default/converge.yml new file mode 100644 index 00000000..9836c7b2 --- /dev/null +++ b/roles/prereq_atlas/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Atlas + ansible.builtin.import_role: + name: prereq_atlas diff --git a/roles/prereq_atlas/molecule/default/create.yml b/roles/prereq_atlas/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_atlas/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_atlas/molecule/default/destroy.yml b/roles/prereq_atlas/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_atlas/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_atlas/molecule/default/molecule.yml b/roles/prereq_atlas/molecule/default/molecule.yml new file mode 100644 index 00000000..0d9c4e0b --- /dev/null +++ b/roles/prereq_atlas/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_atlas-rhel9-4 + Project: Molecule testing for prereq_atlas +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_atlas/molecule/default/requirements.yml b/roles/prereq_atlas/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_atlas/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_atlas/molecule/default/verify.yml b/roles/prereq_atlas/molecule/default/verify.yml new file mode 100644 index 00000000..1db447f7 --- /dev/null +++ b/roles/prereq_atlas/molecule/default/verify.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for atlas user + ansible.builtin.command: grep atlas /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for hadoop group membership + ansible.builtin.command: groups atlas + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("hadoop") + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ atlas_local_accounts }}" diff --git a/roles/prereq_atlas/tasks/main.yml b/roles/prereq_atlas/tasks/main.yml new file mode 100644 index 00000000..2759c6ff --- /dev/null +++ b/roles/prereq_atlas/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ atlas_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ atlas_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_atlas/vars/main.yml b/roles/prereq_atlas/vars/main.yml new file mode 100644 index 00000000..841b17c9 --- /dev/null +++ b/roles/prereq_atlas/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +atlas_local_accounts: + - user: atlas + home: /var/lib/atlas + comment: Atlas + extra_groups: [hadoop] From 9b67d72e75f7eae459bac2de5b96b723d4da30c0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:45 -0400 Subject: [PATCH 04/72] Add Cloudera Manager prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_cloudera_manager/README.md | 66 ++++ .../prereq_cloudera_manager/defaults/main.yml | 18 + .../meta/argument_specs.yml | 35 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 46 +++ roles/prereq_cloudera_manager/tasks/main.yml | 63 ++++ roles/prereq_cloudera_manager/vars/RedHat.yml | 17 + roles/prereq_cloudera_manager/vars/Ubuntu.yml | 17 + .../prereq_cloudera_manager/vars/default.yml | 16 + roles/prereq_cloudera_manager/vars/main.yml | 23 ++ 15 files changed, 925 insertions(+) create mode 100644 roles/prereq_cloudera_manager/README.md create mode 100644 roles/prereq_cloudera_manager/defaults/main.yml create mode 100644 roles/prereq_cloudera_manager/meta/argument_specs.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/converge.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/create.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/destroy.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/molecule.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/prepare.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/requirements.yml create mode 100644 roles/prereq_cloudera_manager/molecule/default/verify.yml create mode 100644 roles/prereq_cloudera_manager/tasks/main.yml create mode 100644 roles/prereq_cloudera_manager/vars/RedHat.yml create mode 100644 roles/prereq_cloudera_manager/vars/Ubuntu.yml create mode 100644 roles/prereq_cloudera_manager/vars/default.yml create mode 100644 roles/prereq_cloudera_manager/vars/main.yml diff --git a/roles/prereq_cloudera_manager/README.md b/roles/prereq_cloudera_manager/README.md new file mode 100644 index 00000000..d0497f6f --- /dev/null +++ b/roles/prereq_cloudera_manager/README.md @@ -0,0 +1,66 @@ +# prereq_cloudera_manager + +Set up for Cloudera Manager + +This role prepares a host for Cloudera Manager usage by performing several foundational setup tasks. It creates the dedicated `cloudera-scm` system user and group, configures the user's home directory and permissions, and can optionally install LDAP client packages for Kerberos support. + +The role will: +- Create the `cloudera-scm` system user and group. +- Configure permissions for the `cloudera-scm` user's home directory (`/var/lib/cloudera-scm`). +- Set up TLS ACLs (Access Control Lists) on the host, if needed by the Cloudera Manager service. +- Optionally install a list of specified LDAP packages, which are often required for Kerberos authentication integration. +- Ensure the Kerberos configuration file (`/etc/krb5.conf`) is properly configured for the Cloudera Manager service. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage system users, groups, and packages. +- The Kerberos configuration file at `kerberos_config_path` must exist on the target host or be managed by another role. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `kerberos_config_path` | `path` | `False` | `/etc/krb5.conf` | Path to the Kerberos configuration file on the target host. | +| `cloudera_manager_ldap_packages` | `list` of `str` | `False` | | List of LDAP packages to install for enabling Kerberos support. If not defined, the role will use default packages based on the OS distribution. | + +# Example Playbook + +```yaml +- hosts: cm_nodes + tasks: + - name: Perform default Cloudera Manager setup + ansible.builtin.import_role: + name: cloudera.exe.prereq_cloudera_manager + # This will create the cloudera-scm user and use the default krb5.conf path. + + - name: Perform Cloudera Manager setup with custom Kerberos and LDAP packages + ansible.builtin.import_role: + name: cloudera.exe.prereq_cloudera_manager + vars: + kerberos_config_path: "/etc/my-custom-krb5.conf" + cloudera_manager_ldap_packages: + - "openldap-clients" + - "nss-pam-ldapd" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_cloudera_manager/defaults/main.yml b/roles/prereq_cloudera_manager/defaults/main.yml new file mode 100644 index 00000000..3fc34d6b --- /dev/null +++ b/roles/prereq_cloudera_manager/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kerberos_config_path: "/etc/krb5.conf" + +# cloudera_manager_ldap_packages: [] diff --git a/roles/prereq_cloudera_manager/meta/argument_specs.yml b/roles/prereq_cloudera_manager/meta/argument_specs.yml new file mode 100644 index 00000000..e3426b0b --- /dev/null +++ b/roles/prereq_cloudera_manager/meta/argument_specs.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Cloudera Manager + description: | + Set up for Cloudera Manager usage, notably, create the local C(cloudera-scm) user, including C(HOME) directory permissions. + Set up TLS ACLs, if needed. + Install LDAP packages for Kerberos, if needed. + author: Cloudera Labs + options: + kerberos_config_path: + description: + - Path to the Kerberos configuration file. + type: path + default: "/etc/krb5.conf" + cloudera_manager_ldap_packages: + description: + - List of LDAP packages to install for Kerberos support. + - If not defined, default packages are specified by OS distribution. + type: list + elements: str diff --git a/roles/prereq_cloudera_manager/molecule/default/converge.yml b/roles/prereq_cloudera_manager/molecule/default/converge.yml new file mode 100644 index 00000000..e34bea54 --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Set up for Cloudera Manager + ansible.builtin.import_role: + name: prereq_cloudera_manager diff --git a/roles/prereq_cloudera_manager/molecule/default/create.yml b/roles/prereq_cloudera_manager/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_cloudera_manager/molecule/default/destroy.yml b/roles/prereq_cloudera_manager/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_cloudera_manager/molecule/default/molecule.yml b/roles/prereq_cloudera_manager/molecule/default/molecule.yml new file mode 100644 index 00000000..db164879 --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_cloudera_manager-rhel9-4 + Project: Molecule testing for prereq_cloudera_manager +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_cloudera_manager/molecule/default/prepare.yml b/roles/prereq_cloudera_manager/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_cloudera_manager/molecule/default/requirements.yml b/roles/prereq_cloudera_manager/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_cloudera_manager/molecule/default/verify.yml b/roles/prereq_cloudera_manager/molecule/default/verify.yml new file mode 100644 index 00000000..0f268f31 --- /dev/null +++ b/roles/prereq_cloudera_manager/molecule/default/verify.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ cloudera_manager_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check HOME directory permissions + ansible.builtin.stat: + path: "{{ account.home }}" + register: __home + failed_when: not __home.stat.exists or __home.stat.mode != account.mode + loop: "{{ cloudera_manager_local_accounts | rejectattr('mode', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ cloudera_manager_local_accounts }}" diff --git a/roles/prereq_cloudera_manager/tasks/main.yml b/roles/prereq_cloudera_manager/tasks/main.yml new file mode 100644 index 00000000..4589225c --- /dev/null +++ b/roles/prereq_cloudera_manager/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ cloudera_manager_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + - mode + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ cloudera_manager_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Get details of Kerberos configuration + ansible.builtin.stat: + path: "{{ kerberos_config_path }}" + register: __krb + +- name: Install LDAP client libraries for Kerberos + when: __krb.stat.exists + ansible.builtin.package: + lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}" + name: "{{ cloudera_manager_ldap_packages | default(ldap_packages) }}" + state: present diff --git a/roles/prereq_cloudera_manager/vars/RedHat.yml b/roles/prereq_cloudera_manager/vars/RedHat.yml new file mode 100644 index 00000000..d2e1023e --- /dev/null +++ b/roles/prereq_cloudera_manager/vars/RedHat.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ldap_packages: + - openldap-clients diff --git a/roles/prereq_cloudera_manager/vars/Ubuntu.yml b/roles/prereq_cloudera_manager/vars/Ubuntu.yml new file mode 100644 index 00000000..8de12aba --- /dev/null +++ b/roles/prereq_cloudera_manager/vars/Ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ldap_packages: + - ldap-utils diff --git a/roles/prereq_cloudera_manager/vars/default.yml b/roles/prereq_cloudera_manager/vars/default.yml new file mode 100644 index 00000000..cadc0ca2 --- /dev/null +++ b/roles/prereq_cloudera_manager/vars/default.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ldap_packages: [] diff --git a/roles/prereq_cloudera_manager/vars/main.yml b/roles/prereq_cloudera_manager/vars/main.yml new file mode 100644 index 00000000..fb69daa8 --- /dev/null +++ b/roles/prereq_cloudera_manager/vars/main.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cloudera_manager_local_accounts: + - user: cloudera-scm + home: /var/lib/cloudera-scm-server + comment: Cloudera Manager + mode: "0770" + keystore_acl: true + key_acl: true + key_password_acl: true From 51612e0db75c495e28366ab80c00e178213d3ce9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:46 -0400 Subject: [PATCH 05/72] Add Cloudera Manager database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_cm_database/README.md | 82 +++++ roles/prereq_cm_database/defaults/main.yml | 26 ++ .../meta/argument_specs.yml | 61 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 60 ++++ .../molecule/default/prepare.yml | 28 ++ .../molecule/default/requirements.yml | 25 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_cm_database/tasks/main.yml | 24 ++ 11 files changed, 871 insertions(+) create mode 100644 roles/prereq_cm_database/README.md create mode 100644 roles/prereq_cm_database/defaults/main.yml create mode 100644 roles/prereq_cm_database/meta/argument_specs.yml create mode 100644 roles/prereq_cm_database/molecule/default/converge.yml create mode 100644 roles/prereq_cm_database/molecule/default/create.yml create mode 100644 roles/prereq_cm_database/molecule/default/destroy.yml create mode 100644 roles/prereq_cm_database/molecule/default/molecule.yml create mode 100644 roles/prereq_cm_database/molecule/default/prepare.yml create mode 100644 roles/prereq_cm_database/molecule/default/requirements.yml create mode 100644 roles/prereq_cm_database/molecule/default/verify.yml create mode 100644 roles/prereq_cm_database/tasks/main.yml diff --git a/roles/prereq_cm_database/README.md b/roles/prereq_cm_database/README.md new file mode 100644 index 00000000..88bc196b --- /dev/null +++ b/roles/prereq_cm_database/README.md @@ -0,0 +1,82 @@ +# prereq_cm_database + +Database and user for Cloudera Manager + +This role creates the necessary database and a dedicated user account for Cloudera Manager to store its metadata. While primarily intended for PostgreSQL as specified in the description, it is designed to be flexible enough to work with other database types such as MySQL and Oracle. The role connects to an existing database server using administrative credentials to perform the setup. + +The role will: +- Connect to the specified database server using the provided `database_admin_user` and `database_admin_password`. +- Create a new database with the name defined by `cloudera_manager_database_name`. +- Create a new database user specified by `cloudera_manager_database_user`. +- Grant ownership and all necessary privileges on the new database to the new user. + +# Requirements + +- A running and accessible database server of the specified `cloudera_manager_database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cloudera_manager_database_user` | `str` | `False` | `scm` | The username for the Cloudera Manager database account. This user will be created and granted ownership of the database. | +| `cloudera_manager_database_type` | `str` | `True` | | The type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `cloudera_manager_database_password` | `str` | `True` | | The password for the Cloudera Manager database user. This password will be used by Cloudera Manager to connect to its database. | +| `cloudera_manager_database_name` | `str` | `False` | `scm` | The name of the database to be created for Cloudera Manager. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_port` | `int` | `False` | - | The port for connecting to the database server. If not specified, the role will use the default port for the specified database type. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Create database and user for Cloudera Manager on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_cm_database + vars: + cloudera_manager_database_type: "postgresql" + cloudera_manager_database_password: "scm_strong_password" # Use Ansible Vault for this + database_admin_user: "postgres" + database_admin_password: "postgres_admin_password" # Use Ansible Vault for this + database_host: "db-server.example.com" + database_port: 5432 # Explicitly set port for clarity + + - name: Create database and user for Cloudera Manager on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_cm_database + vars: + cloudera_manager_database_type: "mysql" + cloudera_manager_database_user: "cm_mysql_user" + cloudera_manager_database_password: "mysql_strong_password" # Use Ansible Vault for this + cloudera_manager_database_name: "cm_metadata" + database_admin_user: "root" + database_admin_password: "mysql_root_password" # Use Ansible Vault for this + database_host: "mysql-server.example.com" + database_port: 3306 +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_cm_database/defaults/main.yml b/roles/prereq_cm_database/defaults/main.yml new file mode 100644 index 00000000..8ad32053 --- /dev/null +++ b/roles/prereq_cm_database/defaults/main.yml @@ -0,0 +1,26 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +cloudera_manager_database_user: scm +cloudera_manager_database_password: "{{ undef(hint='Please define the password to be used for CM database user') }}" +cloudera_manager_database_type: "{{ undef(hint='Please specify the database type') }}" + +cloudera_manager_database_name: scm + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: diff --git a/roles/prereq_cm_database/meta/argument_specs.yml b/roles/prereq_cm_database/meta/argument_specs.yml new file mode 100644 index 00000000..147f7fce --- /dev/null +++ b/roles/prereq_cm_database/meta/argument_specs.yml @@ -0,0 +1,61 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Database and user for Cloudera Manager" + description: + - Creates the database and user for Cloudera Manager on PostgreSQL + author: + - "Jim Enright " + options: + cloudera_manager_database_user: + description: Cloudera Manager database username + type: str + required: false + default: "scm" + cloudera_manager_database_type: + description: + - Database type for the Cloudera Manager server. + choices: + - postgresql + - oracle + - mysql + cloudera_manager_database_password: + description: Password for Cloudera Manager database user + type: str + required: true + cloudera_manager_database_name: + description: Name of Cloudera Manager database + type: str + required: false + default: scm + database_admin_user: + description: The username for the database admin login. + type: str + required: true + database_admin_password: + description: The password for the database admin login. + type: str + required: true + no_log: true + database_host: + description: The hostname or IP address of the database server. + type: str + required: true + database_port: + description: The port for connecting to the database server. + type: int + required: false diff --git a/roles/prereq_cm_database/molecule/default/converge.yml b/roles/prereq_cm_database/molecule/default/converge.yml new file mode 100644 index 00000000..f54a5daf --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Configure Postgres for Cloudera Manager + ansible.builtin.import_role: + name: prereq_cm_database diff --git a/roles/prereq_cm_database/molecule/default/create.yml b/roles/prereq_cm_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_cm_database/molecule/default/destroy.yml b/roles/prereq_cm_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_cm_database/molecule/default/molecule.yml b/roles/prereq_cm_database/molecule/default/molecule.yml new file mode 100644 index 00000000..ae6f5553 --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/molecule.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_cm_database-rhel9-4 + Project: Molecule testing for prereq_cm_database +# Ubuntu 20.04 +# - name: ubuntu20.04.molecule.internal +# image_owner: "099720109477" +# image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* +# instance_type: t3.medium +# boot_wait_seconds: 15 +# vpc_subnet_id: ${TEST_VPC_SUBNET_ID} +# tags: +# Name: molecule-prereq_cm_database-ubuntu20-04 +# Project: Molecule testing for prereq_cm_database +provisioner: + name: ansible + inventory: + group_vars: + all: + cloudera_manager_database_user: scm + cloudera_manager_database_password: "freebird" + cloudera_manager_database_name: scm + cloudera_manager_database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_cm_database/molecule/default/prepare.yml b/roles/prereq_cm_database/molecule/default/prepare.yml new file mode 100644 index 00000000..440ee8ed --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/prepare.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: true + become: true + tasks: + - name: Install PostgreSQL + ansible.builtin.import_role: + name: postgresql_server + vars: + create_database_admin_user: true + # Below are taken from molecule.yml + # database_admin_user: + # database_admin_password: diff --git a/roles/prereq_cm_database/molecule/default/requirements.yml b/roles/prereq_cm_database/molecule/default/requirements.yml new file mode 100644 index 00000000..2adf93f3 --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/requirements.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.utils + - community.general + - ansible.posix + - community.postgresql + +roles: + - geerlingguy.postgresql diff --git a/roles/prereq_cm_database/molecule/default/verify.yml b/roles/prereq_cm_database/molecule/default/verify.yml new file mode 100644 index 00000000..d92f1b32 --- /dev/null +++ b/roles/prereq_cm_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Checks for Postgres database type + when: cloudera_manager_database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'scm';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database amon does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'scm';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User amon does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_cm_database/tasks/main.yml b/roles/prereq_cm_database/tasks/main.yml new file mode 100644 index 00000000..043a9496 --- /dev/null +++ b/roles/prereq_cm_database/tasks/main.yml @@ -0,0 +1,24 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Create Cloudera Manager database + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_type: "{{ cloudera_manager_database_type }}" + database_accounts: + - user: "{{ cloudera_manager_database_user }}" + password: "{{ cloudera_manager_database_password }}" + db: "{{ cloudera_manager_database_name }}" From e0dc2571b6c04228ce71242bb4283d9652820954 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:46 -0400 Subject: [PATCH 06/72] Add database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_database/README.md | 83 +++++ roles/prereq_database/defaults/main.yml | 20 ++ roles/prereq_database/meta/argument_specs.yml | 70 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 51 +++ .../molecule/default/prepare.yml | 32 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 57 +++ roles/prereq_database/tasks/main.yml | 16 + roles/prereq_database/tasks/postgresql.yml | 57 +++ 12 files changed, 924 insertions(+) create mode 100644 roles/prereq_database/README.md create mode 100644 roles/prereq_database/defaults/main.yml create mode 100644 roles/prereq_database/meta/argument_specs.yml create mode 100644 roles/prereq_database/molecule/default/converge.yml create mode 100644 roles/prereq_database/molecule/default/create.yml create mode 100644 roles/prereq_database/molecule/default/destroy.yml create mode 100644 roles/prereq_database/molecule/default/molecule.yml create mode 100644 roles/prereq_database/molecule/default/prepare.yml create mode 100644 roles/prereq_database/molecule/default/requirements.yml create mode 100644 roles/prereq_database/molecule/default/verify.yml create mode 100644 roles/prereq_database/tasks/main.yml create mode 100644 roles/prereq_database/tasks/postgresql.yml diff --git a/roles/prereq_database/README.md b/roles/prereq_database/README.md new file mode 100644 index 00000000..894c4b11 --- /dev/null +++ b/roles/prereq_database/README.md @@ -0,0 +1,83 @@ +# prereq_database + +Create and manage databases and users + +This role configures databases and their associated users in the specified database system. It connects to the database server using administrative credentials and then creates a list of user accounts and databases, granting the correct ownership and permissions. This role is highly flexible and can manage multiple databases and users in a single execution. + +The role will: +- Connect to the specified database server using the provided `database_admin_user` and `database_admin_password`. +- Iterate through the `database_accounts` list to perform the following for each entry: + - Create a new database with the specified name. + - Create a new user account with the specified username and password. + - Grant ownership of the new database to the specified owner (defaults to the created user). +- Ensure that all privileges are correctly assigned for the new users and databases. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_admin_user` | `str` | `True` | | The username for the database admin login. | +| `database_admin_password` | `str` | `True` | | The password for the database admin login. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_port` | `int` | `False` | - | The port for connecting to the database server. If not specified, the role will use the default port for the specified database type. | +| `database_accounts` | `list` of `dict` | `True` | | A list of database accounts to create and manage. Each item in the list is a dictionary with the following keys. | +|     `db` | `str` | `True` | | The name of the database to create. | +|     `user` | `str` | `True` | | The name of the database user. | +|     `password` | `str` | `True` | | The password for the database user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +|     `owner` | `str` | `False` | `user` | The name of the database user who should own the database. If not specified, the `user` will be set as the owner. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Manage databases and users on a PostgreSQL server + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + database_port: 5432 + database_accounts: + - db: "scm" + user: "scm_user" + password: "scm_password_here" # Use Ansible Vault for this + - db: "ranger" + user: "ranger_user" + password: "ranger_password_here" + owner: "ranger_user" # Explicitly setting owner + - db: "hive" + user: "hive_user" + password: "hive_password_here" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_database/defaults/main.yml b/roles/prereq_database/defaults/main.yml new file mode 100644 index 00000000..2dfbdd96 --- /dev/null +++ b/roles/prereq_database/defaults/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +database_accounts: [] +database_type: "{{ undef(hint='Please define database type') }}" +database_admin_user: "{{ undef(hint='Please define database admin user') }}" +database_admin_password: "{{ undef(hint='Please define databaser admin password') }}" +# database_port: 5432 diff --git a/roles/prereq_database/meta/argument_specs.yml b/roles/prereq_database/meta/argument_specs.yml new file mode 100644 index 00000000..ca8c4bd9 --- /dev/null +++ b/roles/prereq_database/meta/argument_specs.yml @@ -0,0 +1,70 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Create and manage databases and users + description: | + This role configures databases and their associated users in the specified database system. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_admin_user: + description: The username for the database admin login. + type: str + required: true + database_admin_password: + description: The password for the database admin login. + type: str + required: true + no_log: true + database_host: + description: The hostname or IP address of the database server. + type: str + required: true + database_port: + description: The port for connecting to the database server. + type: int + required: false + database_accounts: + description: A list of database accounts to create and manage. + type: list + elements: dict + required: true + options: + db: + description: The name of the database to create. + type: str + required: true + user: + description: The name of the database user. + type: str + required: true + password: + description: The password for the database user. + type: str + required: true + no_log: true + owner: + description: The name of the database user owning the database. Defaults to O(user). + type: str + required: false diff --git a/roles/prereq_database/molecule/default/converge.yml b/roles/prereq_database/molecule/default/converge.yml new file mode 100644 index 00000000..8ea91dc4 --- /dev/null +++ b/roles/prereq_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: false + tasks: + - name: Create and manage databases and users + ansible.builtin.import_role: + name: prereq_database diff --git a/roles/prereq_database/molecule/default/create.yml b/roles/prereq_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_database/molecule/default/destroy.yml b/roles/prereq_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_database/molecule/default/molecule.yml b/roles/prereq_database/molecule/default/molecule.yml new file mode 100644 index 00000000..cfb519f2 --- /dev/null +++ b/roles/prereq_database/molecule/default/molecule.yml @@ -0,0 +1,51 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + security_groups: ${TEST_VPC_SECURITY_GROUP} + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_databases-rhel9-4 + Project: Molecule testing for prereq_databases +provisioner: + name: ansible + inventory: + group_vars: + all: + database_admin_user: molecule + database_admin_password: molecule + database_type: postgresql + database_host: localhost + database_accounts: + - db: testdb + user: test + password: test diff --git a/roles/prereq_database/molecule/default/prepare.yml b/roles/prereq_database/molecule/default/prepare.yml new file mode 100644 index 00000000..686e92ed --- /dev/null +++ b/roles/prereq_database/molecule/default/prepare.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare database service and superuser + hosts: all + gather_facts: true + become: true + tasks: + - block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres + when: database_type == 'postgresql' diff --git a/roles/prereq_database/molecule/default/requirements.yml b/roles/prereq_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_database/molecule/default/verify.yml b/roles/prereq_database/molecule/default/verify.yml new file mode 100644 index 00000000..5141308f --- /dev/null +++ b/roles/prereq_database/molecule/default/verify.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = '{{ account.db }}';" + loop: "{{ database_accounts }}" + loop_control: + loop_var: account + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database {{ item.account.db }} does not exist!" + loop: "{{ db_check.results }}" + when: item.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = '{{ account.user }}';" + loop: "{{ database_accounts }}" + loop_control: + loop_var: account + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User {{ item.account.user }} does not exist!" + loop: "{{ user_check.results }}" + when: item.query_result | length == 0 diff --git a/roles/prereq_database/tasks/main.yml b/roles/prereq_database/tasks/main.yml new file mode 100644 index 00000000..1a00c479 --- /dev/null +++ b/roles/prereq_database/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Load database tasks for the specified database + ansible.builtin.include_tasks: "{{ database_type }}.yml" diff --git a/roles/prereq_database/tasks/postgresql.yml b/roles/prereq_database/tasks/postgresql.yml new file mode 100644 index 00000000..52bd3da3 --- /dev/null +++ b/roles/prereq_database/tasks/postgresql.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create PostgreSQL database + community.postgresql.postgresql_db: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + name: "{{ account.db }}" + port: "{{ database_port | default(omit) }}" + encoding: UTF-8 + loop: "{{ database_accounts }}" + loop_control: + loop_var: account + label: "{{ account.db }}" + no_log: true + +- name: Create PostgreSQL role + community.postgresql.postgresql_user: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + login_db: "{{ account.db }}" + name: "{{ account.user }}" + password: "{{ account.password }}" + loop: "{{ database_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + no_log: true + +- name: Set owner for PostgreSQL database + community.postgresql.postgresql_owner: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + login_db: "{{ account.db }}" + new_owner: "{{ account.user }}" + obj_type: database + obj_name: "{{ account.db }}" + loop: "{{ database_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + no_log: true From ed0aa847f6172ecc27de9fabccc47b6c3776f9a6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:47 -0400 Subject: [PATCH 07/72] Add Dataviz prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_dataviz/README.md | 52 +++ roles/prereq_dataviz/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_dataviz/tasks/main.yml | 39 ++ roles/prereq_dataviz/vars/main.yml | 20 ++ 11 files changed, 793 insertions(+) create mode 100644 roles/prereq_dataviz/README.md create mode 100644 roles/prereq_dataviz/meta/argument_specs.yml create mode 100644 roles/prereq_dataviz/molecule/default/converge.yml create mode 100644 roles/prereq_dataviz/molecule/default/create.yml create mode 100644 roles/prereq_dataviz/molecule/default/destroy.yml create mode 100644 roles/prereq_dataviz/molecule/default/molecule.yml create mode 100644 roles/prereq_dataviz/molecule/default/prepare.yml create mode 100644 roles/prereq_dataviz/molecule/default/requirements.yml create mode 100644 roles/prereq_dataviz/molecule/default/verify.yml create mode 100644 roles/prereq_dataviz/tasks/main.yml create mode 100644 roles/prereq_dataviz/vars/main.yml diff --git a/roles/prereq_dataviz/README.md b/roles/prereq_dataviz/README.md new file mode 100644 index 00000000..4199d013 --- /dev/null +++ b/roles/prereq_dataviz/README.md @@ -0,0 +1,52 @@ +# prereq_dataviz + +Set up for Dataviz + +This role prepares a host for Dataviz usage by creating a dedicated system user and group named `dataviz`. This user is essential for running Dataviz processes with appropriate permissions and isolation. + +The role will: +- Create the `dataviz` system user and group. +- Configure home directories and other necessary local paths for the `dataviz` user, if required. +- Ensure appropriate permissions are set for files and directories related to Dataviz. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: dataviz_nodes + tasks: + - name: Set up the dataviz user and environment + ansible.builtin.import_role: + name: dataviz_setup +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_dataviz/meta/argument_specs.yml b/roles/prereq_dataviz/meta/argument_specs.yml new file mode 100644 index 00000000..8f4d21ef --- /dev/null +++ b/roles/prereq_dataviz/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Dataviz + description: | + Set up for Dataviz usage, notably, create the local C(dataviz) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_dataviz/molecule/default/converge.yml b/roles/prereq_dataviz/molecule/default/converge.yml new file mode 100644 index 00000000..2ee435b4 --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Dataviz + ansible.builtin.import_role: + name: prereq_dataviz diff --git a/roles/prereq_dataviz/molecule/default/create.yml b/roles/prereq_dataviz/molecule/default/create.yml new file mode 100644 index 00000000..e40c7f7a --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_dataviz/molecule/default/destroy.yml b/roles/prereq_dataviz/molecule/default/destroy.yml new file mode 100644 index 00000000..8d3a4863 --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_dataviz/molecule/default/molecule.yml b/roles/prereq_dataviz/molecule/default/molecule.yml new file mode 100644 index 00000000..44cc95d8 --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_dataviz-rhel9-4 + Project: Molecule testing for prereq_dataviz +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_dataviz/molecule/default/prepare.yml b/roles/prereq_dataviz/molecule/default/prepare.yml new file mode 100644 index 00000000..62e044c1 --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_dataviz/molecule/default/requirements.yml b/roles/prereq_dataviz/molecule/default/requirements.yml new file mode 100644 index 00000000..8cfddb54 --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_dataviz/molecule/default/verify.yml b/roles/prereq_dataviz/molecule/default/verify.yml new file mode 100644 index 00000000..a0012a3a --- /dev/null +++ b/roles/prereq_dataviz/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for dataviz users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ dataviz_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ dataviz_local_accounts }}" diff --git a/roles/prereq_dataviz/tasks/main.yml b/roles/prereq_dataviz/tasks/main.yml new file mode 100644 index 00000000..77c3979b --- /dev/null +++ b/roles/prereq_dataviz/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ dataviz_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ dataviz_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_dataviz/vars/main.yml b/roles/prereq_dataviz/vars/main.yml new file mode 100644 index 00000000..a83e8744 --- /dev/null +++ b/roles/prereq_dataviz/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dataviz_local_accounts: + - user: dataviz + home: /var/lib/dataviz + comment: dataviz + keystore_acl: true From 0d88415e90e74da5f57a6387b06d6c8cece3ad18 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:48 -0400 Subject: [PATCH 08/72] Add Dataviz database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_dataviz_database/README.md | 79 ++++ .../prereq_dataviz_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 32 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_dataviz_database/tasks/main.yml | 24 ++ 11 files changed, 852 insertions(+) create mode 100644 roles/prereq_dataviz_database/README.md create mode 100644 roles/prereq_dataviz_database/defaults/main.yml create mode 100644 roles/prereq_dataviz_database/meta/argument_specs.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/converge.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/create.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/destroy.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/molecule.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/prepare.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/requirements.yml create mode 100644 roles/prereq_dataviz_database/molecule/default/verify.yml create mode 100644 roles/prereq_dataviz_database/tasks/main.yml diff --git a/roles/prereq_dataviz_database/README.md b/roles/prereq_dataviz_database/README.md new file mode 100644 index 00000000..529bcf22 --- /dev/null +++ b/roles/prereq_dataviz_database/README.md @@ -0,0 +1,79 @@ +# prereq_dataviz_database + +Set up database and user accounts for Dataviz + +This role automates the setup of a database and its associated user accounts specifically for Cloudera's Dataviz service. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `dataviz_database`. +- Create a new database user specified by `dataviz_username` with the password from `dataviz_password`. +- Grant ownership and all necessary privileges to the `dataviz_username` for the new database. +- Ensure the database is configured correctly for Dataviz operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `dataviz_username` | `str` | `False` | `dataviz` | The username for the Dataviz database user. This user will also be the owner of the database. | +| `dataviz_password` | `str` | `False` | `dataviz` | The password for the Dataviz database user. It is highly recommended to override this default in production. | +| `dataviz_database` | `str` | `False` | `dataviz` | The name of the database to be created for Dataviz. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Dataviz database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_dataviz_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Dataviz database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_dataviz_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + dataviz_username: "my_dviz_user" + dataviz_password: "a_strong_dviz_password" + dataviz_database: "my_dviz_db" +``` + +# License + +``` +Copyright 2025 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_dataviz_database/defaults/main.yml b/roles/prereq_dataviz_database/defaults/main.yml new file mode 100644 index 00000000..e4014ea1 --- /dev/null +++ b/roles/prereq_dataviz_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +dataviz_username: dataviz +dataviz_password: dataviz +dataviz_database: dataviz diff --git a/roles/prereq_dataviz_database/meta/argument_specs.yml b/roles/prereq_dataviz_database/meta/argument_specs.yml new file mode 100644 index 00000000..3d0a610f --- /dev/null +++ b/roles/prereq_dataviz_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Dataviz + description: + - Set up the Dataviz database and its associated user accounts, ensuring proper configuration for Dataviz operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `dataviz_username`, + `dataviz_password`, and `dataviz_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + dataviz_username: + description: The username for the Dataviz database user and owner of the database. + type: str + required: false + default: dataviz + dataviz_password: + description: The password for the Dataviz database user. + type: str + required: false + default: dataviz + dataviz_database: + description: The name of the database to be created for Dataviz. + type: str + required: false + default: dataviz diff --git a/roles/prereq_dataviz_database/molecule/default/converge.yml b/roles/prereq_dataviz_database/molecule/default/converge.yml new file mode 100644 index 00000000..f890a5ef --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Dataviz database and configure its associated user account. + ansible.builtin.import_role: + name: prereq_dataviz_database diff --git a/roles/prereq_dataviz_database/molecule/default/create.yml b/roles/prereq_dataviz_database/molecule/default/create.yml new file mode 100644 index 00000000..e40c7f7a --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_dataviz_database/molecule/default/destroy.yml b/roles/prereq_dataviz_database/molecule/default/destroy.yml new file mode 100644 index 00000000..8d3a4863 --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_dataviz_database/molecule/default/molecule.yml b/roles/prereq_dataviz_database/molecule/default/molecule.yml new file mode 100644 index 00000000..8a0e8c6b --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_dataviz_database-rhel9-4 + Project: Molecule testing for prereq_dataviz_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_dataviz_database/molecule/default/prepare.yml b/roles/prereq_dataviz_database/molecule/default/prepare.yml new file mode 100644 index 00000000..5a827558 --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/prepare.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres + when: database_type == 'postgresql' diff --git a/roles/prereq_dataviz_database/molecule/default/requirements.yml b/roles/prereq_dataviz_database/molecule/default/requirements.yml new file mode 100644 index 00000000..3a5c5a35 --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_dataviz_database/molecule/default/verify.yml b/roles/prereq_dataviz_database/molecule/default/verify.yml new file mode 100644 index 00000000..5e5b3940 --- /dev/null +++ b/roles/prereq_dataviz_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'dataviz';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database dataviz does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'dataviz';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User dataviz does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_dataviz_database/tasks/main.yml b/roles/prereq_dataviz_database/tasks/main.yml new file mode 100644 index 00000000..56350b97 --- /dev/null +++ b/roles/prereq_dataviz_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: prereq_database + vars: + database_accounts: "{{ dataviz_details }}" + dataviz_details: + - user: "{{ dataviz_username }}" + password: "{{ dataviz_password }}" + db: "{{ dataviz_database }}" + no_log: true From ce8388b010a8e645d1372bf3c41b191d5a7c7644 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:48 -0400 Subject: [PATCH 09/72] Add Apache Druid prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_druid/README.md | 52 +++ roles/prereq_druid/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_druid/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_druid/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_druid/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_druid/molecule/default/verify.yml | 38 ++ roles/prereq_druid/tasks/main.yml | 39 ++ roles/prereq_druid/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_druid/README.md create mode 100644 roles/prereq_druid/meta/argument_specs.yml create mode 100644 roles/prereq_druid/molecule/default/converge.yml create mode 100644 roles/prereq_druid/molecule/default/create.yml create mode 100644 roles/prereq_druid/molecule/default/destroy.yml create mode 100644 roles/prereq_druid/molecule/default/molecule.yml create mode 100644 roles/prereq_druid/molecule/default/prepare.yml create mode 100644 roles/prereq_druid/molecule/default/requirements.yml create mode 100644 roles/prereq_druid/molecule/default/verify.yml create mode 100644 roles/prereq_druid/tasks/main.yml create mode 100644 roles/prereq_druid/vars/main.yml diff --git a/roles/prereq_druid/README.md b/roles/prereq_druid/README.md new file mode 100644 index 00000000..3d9b4fd3 --- /dev/null +++ b/roles/prereq_druid/README.md @@ -0,0 +1,52 @@ +# prereq_druid + +Set up for Apache Druid + +This role prepares a host for Apache Druid usage by creating a dedicated system user and group named `druid`. This user is essential for running Apache Druid processes with appropriate permissions and isolation. + +The role will: +- Create the `druid` system user and group. +- Configure home directories and other necessary local paths for the `druid` user, if required. +- Ensure appropriate permissions are set for files and directories related to Apache Druid. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: druid_nodes + tasks: + - name: Set up the apache druid user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_druid +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_druid/meta/argument_specs.yml b/roles/prereq_druid/meta/argument_specs.yml new file mode 100644 index 00000000..d1b3105a --- /dev/null +++ b/roles/prereq_druid/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Druid + description: | + Set up for Druid usage, notably, create the local C(druid) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_druid/molecule/default/converge.yml b/roles/prereq_druid/molecule/default/converge.yml new file mode 100644 index 00000000..2be053b0 --- /dev/null +++ b/roles/prereq_druid/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Druid + ansible.builtin.import_role: + name: prereq_druid diff --git a/roles/prereq_druid/molecule/default/create.yml b/roles/prereq_druid/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_druid/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_druid/molecule/default/destroy.yml b/roles/prereq_druid/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_druid/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_druid/molecule/default/molecule.yml b/roles/prereq_druid/molecule/default/molecule.yml new file mode 100644 index 00000000..7a59bdcf --- /dev/null +++ b/roles/prereq_druid/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_druid-rhel9-4 + Project: Molecule testing for prereq_druid +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_druid/molecule/default/prepare.yml b/roles/prereq_druid/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_druid/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_druid/molecule/default/requirements.yml b/roles/prereq_druid/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_druid/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_druid/molecule/default/verify.yml b/roles/prereq_druid/molecule/default/verify.yml new file mode 100644 index 00000000..ea3f67b0 --- /dev/null +++ b/roles/prereq_druid/molecule/default/verify.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for druid user + ansible.builtin.command: grep druid /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for hadoop group membership + ansible.builtin.command: groups druid + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("hadoop") + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ druid_local_accounts }}" diff --git a/roles/prereq_druid/tasks/main.yml b/roles/prereq_druid/tasks/main.yml new file mode 100644 index 00000000..2f165f73 --- /dev/null +++ b/roles/prereq_druid/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ druid_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ druid_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_druid/vars/main.yml b/roles/prereq_druid/vars/main.yml new file mode 100644 index 00000000..577a3bae --- /dev/null +++ b/roles/prereq_druid/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +druid_local_accounts: + - user: druid + home: /var/lib/druid + comment: Druid + extra_groups: [hadoop] From 5d06df9952fba99982e615767ac9e68492867a6d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:49 -0400 Subject: [PATCH 10/72] Add ECS prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_dataviz_database/tasks/main.yml | 2 +- roles/prereq_ecs/README.md | 52 +++ roles/prereq_ecs/files/networkmanager.conf | 2 + roles/prereq_ecs/handlers/main.yml | 33 ++ roles/prereq_ecs/meta/argument_specs.yml | 23 ++ .../prereq_ecs/molecule/default/converge.yml | 23 ++ roles/prereq_ecs/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_ecs/molecule/default/destroy.yml | 157 ++++++++ .../prereq_ecs/molecule/default/molecule.yml | 49 +++ roles/prereq_ecs/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_ecs/molecule/default/verify.yml | 54 +++ roles/prereq_ecs/tasks/RedHat-8-iptables.yml | 64 ++++ roles/prereq_ecs/tasks/RedHat-9-iptables.yml | 50 +++ roles/prereq_ecs/tasks/default-iptables.yml | 13 + roles/prereq_ecs/tasks/main.yml | 83 +++++ roles/prereq_ecs/vars/RedHat-8.yml | 21 ++ roles/prereq_ecs/vars/RedHat-9.yml | 20 ++ 18 files changed, 1040 insertions(+), 1 deletion(-) create mode 100644 roles/prereq_ecs/README.md create mode 100644 roles/prereq_ecs/files/networkmanager.conf create mode 100644 roles/prereq_ecs/handlers/main.yml create mode 100644 roles/prereq_ecs/meta/argument_specs.yml create mode 100644 roles/prereq_ecs/molecule/default/converge.yml create mode 100644 roles/prereq_ecs/molecule/default/create.yml create mode 100644 roles/prereq_ecs/molecule/default/destroy.yml create mode 100644 roles/prereq_ecs/molecule/default/molecule.yml create mode 100644 roles/prereq_ecs/molecule/default/prepare.yml create mode 100644 roles/prereq_ecs/molecule/default/requirements.yml create mode 100644 roles/prereq_ecs/molecule/default/verify.yml create mode 100644 roles/prereq_ecs/tasks/RedHat-8-iptables.yml create mode 100644 roles/prereq_ecs/tasks/RedHat-9-iptables.yml create mode 100644 roles/prereq_ecs/tasks/default-iptables.yml create mode 100644 roles/prereq_ecs/tasks/main.yml create mode 100644 roles/prereq_ecs/vars/RedHat-8.yml create mode 100644 roles/prereq_ecs/vars/RedHat-9.yml diff --git a/roles/prereq_dataviz_database/tasks/main.yml b/roles/prereq_dataviz_database/tasks/main.yml index 56350b97..dbeee725 100644 --- a/roles/prereq_dataviz_database/tasks/main.yml +++ b/roles/prereq_dataviz_database/tasks/main.yml @@ -14,7 +14,7 @@ # limitations under the License. - name: Provision databases and user accounts ansible.builtin.import_role: - name: prereq_database + name: cloudera.exe.prereq_database vars: database_accounts: "{{ dataviz_details }}" dataviz_details: diff --git a/roles/prereq_ecs/README.md b/roles/prereq_ecs/README.md new file mode 100644 index 00000000..492424b4 --- /dev/null +++ b/roles/prereq_ecs/README.md @@ -0,0 +1,52 @@ +# prereq_ecs + +Set up for ECS + +This role prepares a host for Cloudera's ECS (Embedded Container Service) usage by creating the required local users and configuring the firewall and network settings. It ensures that the host's environment is properly configured to support ECS components and operations, including user permissions and network security rules. + +The role will: +- Create the necessary system users and groups for ECS, based on a list provided by the `prereq_cloudera_manager` role. +- Configure firewall rules to allow traffic required by ECS components. +- Set up networking configurations to ensure proper communication within the ECS environment. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage system users, firewall rules, and network configurations. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: ecs_nodes + tasks: + - name: Set up the host for ECS usage + ansible.builtin.import_role: + name: cloudera.exe.prereq_ecs +``` + +## License + +``` +Copyright 2025 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ecs/files/networkmanager.conf b/roles/prereq_ecs/files/networkmanager.conf new file mode 100644 index 00000000..13d2a834 --- /dev/null +++ b/roles/prereq_ecs/files/networkmanager.conf @@ -0,0 +1,2 @@ +[keyfile] +unmanaged-devices=interface-name:cali*;interface-name:flannel* \ No newline at end of file diff --git a/roles/prereq_ecs/handlers/main.yml b/roles/prereq_ecs/handlers/main.yml new file mode 100644 index 00000000..dec63c31 --- /dev/null +++ b/roles/prereq_ecs/handlers/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# - name: Flush iptables +# ansible.builtin.iptables: +# flush: yes +# table: "{{ __iptables_flush_item }}" +# loop: +# - filter +# - nat +# - mangle +# - raw +# - security +# loop_control: +# loop_var: __iptables_flush_item + +- name: Restart network + ansible.builtin.service: + name: "{{ network_service }}" + state: restarted + daemon_reload: true diff --git a/roles/prereq_ecs/meta/argument_specs.yml b/roles/prereq_ecs/meta/argument_specs.yml new file mode 100644 index 00000000..654d04ed --- /dev/null +++ b/roles/prereq_ecs/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for ECS + description: + - Set up for ECS usage including creation of required local users and configuration of firewall and networking configuration. + - The list of local users are taken from the M(cloudera.exe.prereq_cloudera_manager) role. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_ecs/molecule/default/converge.yml b/roles/prereq_ecs/molecule/default/converge.yml new file mode 100644 index 00000000..3d407441 --- /dev/null +++ b/roles/prereq_ecs/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for ECS + ansible.builtin.import_role: + name: prereq_ecs diff --git a/roles/prereq_ecs/molecule/default/create.yml b/roles/prereq_ecs/molecule/default/create.yml new file mode 100644 index 00000000..e40c7f7a --- /dev/null +++ b/roles/prereq_ecs/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ecs/molecule/default/destroy.yml b/roles/prereq_ecs/molecule/default/destroy.yml new file mode 100644 index 00000000..8d3a4863 --- /dev/null +++ b/roles/prereq_ecs/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ecs/molecule/default/molecule.yml b/roles/prereq_ecs/molecule/default/molecule.yml new file mode 100644 index 00000000..3571afe3 --- /dev/null +++ b/roles/prereq_ecs/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_ecs-rhel9-4 + Project: Molecule testing for prereq_ecs +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_ecs/molecule/default/prepare.yml b/roles/prereq_ecs/molecule/default/prepare.yml new file mode 100644 index 00000000..62e044c1 --- /dev/null +++ b/roles/prereq_ecs/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_ecs/molecule/default/requirements.yml b/roles/prereq_ecs/molecule/default/requirements.yml new file mode 100644 index 00000000..8cfddb54 --- /dev/null +++ b/roles/prereq_ecs/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_ecs/molecule/default/verify.yml b/roles/prereq_ecs/molecule/default/verify.yml new file mode 100644 index 00000000..e9c8df2a --- /dev/null +++ b/roles/prereq_ecs/molecule/default/verify.yml @@ -0,0 +1,54 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + # vars_files: "../../vars/main.yml" + pre_tasks: + - name: Include local_accounts from Cloudera Manager role + ansible.builtin.include_role: + name: prereq_cloudera_manager + public: true + rolespec_validate: false + tasks_from: no_op.yml + tasks: + - name: Check for ECS users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ cloudera_manager_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Confirm symlinks exists + when: account.symlink is defined + ansible.builtin.stat: + path: "{{ account.symlink }}" + register: _symlink + failed_when: not _symlink.stat.islnk + loop: "{{ cloudera_manager_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ cloudera_manager_local_accounts }}" diff --git a/roles/prereq_ecs/tasks/RedHat-8-iptables.yml b/roles/prereq_ecs/tasks/RedHat-8-iptables.yml new file mode 100644 index 00000000..0155cb6f --- /dev/null +++ b/roles/prereq_ecs/tasks/RedHat-8-iptables.yml @@ -0,0 +1,64 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather the package facts + ansible.builtin.package_facts: + manager: auto + +- name: Install iptables, using rpm option tsflags=noscripts + when: ansible_facts.packages[iptables_package] is not defined + ansible.builtin.command: "dnf install -y {{ iptables_package }} --setopt=tsflags=noscripts" + changed_when: false # TODO Revisit + +- name: Flush iptables + ansible.builtin.iptables: + flush: true + table: "{{ __iptables_flush_item }}" + loop: + - filter + - nat + - mangle + - raw + - security + loop_control: + loop_var: __iptables_flush_item + +- name: Gather services facts + ansible.builtin.service_facts: + +# Required as per https://docs.rke2.io/known_issues +- name: Set NetworkManager to ignore any ECS calico & flannel interfaces + ansible.builtin.copy: + src: networkmanager.conf + dest: /etc/NetworkManager/conf.d/rke2-canal.config + owner: root + group: root + mode: "0644" + when: "'NetworkManager.service' in ansible_facts.services" + notify: Restart network diff --git a/roles/prereq_ecs/tasks/RedHat-9-iptables.yml b/roles/prereq_ecs/tasks/RedHat-9-iptables.yml new file mode 100644 index 00000000..2fc92daf --- /dev/null +++ b/roles/prereq_ecs/tasks/RedHat-9-iptables.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather the package facts + ansible.builtin.package_facts: + manager: auto + +- name: Install iptables, using rpm option tsflags=noscripts + when: ansible_facts.packages[iptables_package] is not defined + ansible.builtin.command: "dnf install -y {{ iptables_package }} --setopt=tsflags=noscripts" + changed_when: false # TODO Revisit + +- name: Flush iptables + ansible.builtin.command: "iptables-nft --flush -t {{ __iptables_flush_item }}" + loop: + - filter + - nat + - mangle + - raw + - security + loop_control: + loop_var: __iptables_flush_item + changed_when: false + tags: molecule-idempotence-notest + +- name: Gather services facts + ansible.builtin.service_facts: + +# Required as per https://docs.rke2.io/known_issues +- name: Set NetworkManager to ignore any ECS calico & flannel interfaces + ansible.builtin.copy: + src: networkmanager.conf + dest: /etc/NetworkManager/conf.d/rke2-canal.config + owner: root + group: root + mode: "0644" + when: "'NetworkManager.service' in ansible_facts.services" + notify: Restart network diff --git a/roles/prereq_ecs/tasks/default-iptables.yml b/roles/prereq_ecs/tasks/default-iptables.yml new file mode 100644 index 00000000..c26b46e1 --- /dev/null +++ b/roles/prereq_ecs/tasks/default-iptables.yml @@ -0,0 +1,13 @@ +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/roles/prereq_ecs/tasks/main.yml b/roles/prereq_ecs/tasks/main.yml new file mode 100644 index 00000000..be11a75f --- /dev/null +++ b/roles/prereq_ecs/tasks/main.yml @@ -0,0 +1,83 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Include local_accounts from Cloudera Manager role + ansible.builtin.include_role: + name: cloudera.exe.prereq_cloudera_manager + public: true + rolespec_validate: false + tasks_from: no_op.yml + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + rolespec_validate: false + vars: + local_accounts: "{{ cloudera_manager_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + - mode + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + rolespec_validate: false + vars: + acl_user_accounts: "{{ cloudera_manager_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl + +- name: Gather host distribution details + ansible.builtin.setup: + gather_subset: distribution + +- name: Load distribution variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install necessary packages for ECS + ansible.builtin.package: + name: "{{ __package_item }}" + state: present + loop: "{{ ecs_required_packages }}" + loop_control: + loop_var: __package_item + +- name: Install and Configure iptables + ansible.builtin.include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}-iptables.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}-iptables.yml" + - "{{ ansible_facts['distribution'] }}-iptables.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}-iptables.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}-iptables.yml" + - "{{ ansible_facts['os_family'] }}-iptables.yml" + - "default-iptables.yml" diff --git a/roles/prereq_ecs/vars/RedHat-8.yml b/roles/prereq_ecs/vars/RedHat-8.yml new file mode 100644 index 00000000..a6bc498e --- /dev/null +++ b/roles/prereq_ecs/vars/RedHat-8.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ecs_required_packages: + - nfs-utils + - iscsi-initiator-utils + +network_service: NetworkManager +iptables_package: iptables diff --git a/roles/prereq_ecs/vars/RedHat-9.yml b/roles/prereq_ecs/vars/RedHat-9.yml new file mode 100644 index 00000000..3cfc62a7 --- /dev/null +++ b/roles/prereq_ecs/vars/RedHat-9.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ecs_required_packages: + - nfs-utils + - iscsi-initiator-utils + +network_service: NetworkManager +iptables_package: iptables-utils From 78fd95c4a2acd71279e658e5dd775f21f925661f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:50 -0400 Subject: [PATCH 11/72] Add firewall prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_firewall/README.md | 71 ++++ roles/prereq_firewall/defaults/main.yml | 18 + roles/prereq_firewall/handlers/main.yml | 16 + roles/prereq_firewall/meta/argument_specs.yml | 36 ++ .../molecule/bkup-dir/converge.yml | 25 ++ .../molecule/bkup-dir/create.yml | 336 ++++++++++++++++++ .../molecule/bkup-dir/destroy.yml | 157 ++++++++ .../molecule/bkup-dir/molecule.yml | 45 +++ .../molecule/bkup-dir/requirements.yml | 21 ++ .../molecule/bkup-dir/verify.yml | 28 ++ .../molecule/bkup-format/converge.yml | 25 ++ .../molecule/bkup-format/create.yml | 336 ++++++++++++++++++ .../molecule/bkup-format/destroy.yml | 157 ++++++++ .../molecule/bkup-format/molecule.yml | 45 +++ .../molecule/bkup-format/requirements.yml | 21 ++ .../molecule/bkup-format/verify.yml | 28 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 41 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 28 ++ .../molecule/no-bkup/converge.yml | 25 ++ .../molecule/no-bkup/create.yml | 336 ++++++++++++++++++ .../molecule/no-bkup/destroy.yml | 157 ++++++++ .../molecule/no-bkup/molecule.yml | 45 +++ .../molecule/no-bkup/requirements.yml | 21 ++ .../molecule/no-bkup/verify.yml | 28 ++ .../prereq_firewall/tasks/RedHat-firewall.yml | 24 ++ .../tasks/default-firewall.yml | 18 + roles/prereq_firewall/tasks/main.yml | 63 ++++ roles/prereq_firewall/tests/inventory | 15 + roles/prereq_firewall/tests/test.yml | 20 ++ roles/prereq_firewall/vars/RedHat.yml | 19 + roles/prereq_firewall/vars/default.yml | 20 ++ 35 files changed, 2762 insertions(+) create mode 100644 roles/prereq_firewall/README.md create mode 100644 roles/prereq_firewall/defaults/main.yml create mode 100644 roles/prereq_firewall/handlers/main.yml create mode 100644 roles/prereq_firewall/meta/argument_specs.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/converge.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/create.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/destroy.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/molecule.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/requirements.yml create mode 100644 roles/prereq_firewall/molecule/bkup-dir/verify.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/converge.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/create.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/destroy.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/molecule.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/requirements.yml create mode 100644 roles/prereq_firewall/molecule/bkup-format/verify.yml create mode 100644 roles/prereq_firewall/molecule/default/converge.yml create mode 100644 roles/prereq_firewall/molecule/default/create.yml create mode 100644 roles/prereq_firewall/molecule/default/destroy.yml create mode 100644 roles/prereq_firewall/molecule/default/molecule.yml create mode 100644 roles/prereq_firewall/molecule/default/requirements.yml create mode 100644 roles/prereq_firewall/molecule/default/verify.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/converge.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/create.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/destroy.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/molecule.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/requirements.yml create mode 100644 roles/prereq_firewall/molecule/no-bkup/verify.yml create mode 100644 roles/prereq_firewall/tasks/RedHat-firewall.yml create mode 100644 roles/prereq_firewall/tasks/default-firewall.yml create mode 100644 roles/prereq_firewall/tasks/main.yml create mode 100644 roles/prereq_firewall/tests/inventory create mode 100644 roles/prereq_firewall/tests/test.yml create mode 100644 roles/prereq_firewall/vars/RedHat.yml create mode 100644 roles/prereq_firewall/vars/default.yml diff --git a/roles/prereq_firewall/README.md b/roles/prereq_firewall/README.md new file mode 100644 index 00000000..d1da8f2c --- /dev/null +++ b/roles/prereq_firewall/README.md @@ -0,0 +1,71 @@ +# prereq_firewall + +Disable firewalls + +This role ensures that firewalls on the target host are disabled. It can intelligently back up existing firewall rules (for both IPv4 and IPv6) before disabling the service. This allows for a straightforward restoration of the original firewall state if needed. + +The role will: +- Check for and stop active firewall services, such as `firewalld` or `iptables`. +- Create a backup of the current `iptables` rules for both IPv4 and IPv6 in a timestamped format. The backup directory defaults to the Ansible user's home directory. Backup file naming is `iptables-rules-[ipv4|ipv6].TIMESTAMP`, where `TIMESTAMP` is UTC. +- Disable firewall services from starting on boot. +- Flush all existing `iptables` rules to ensure an open network. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage system services and firewall rules. +- The target host must have `iptables` installed for the backup functionality to work. The role will likely also handle services like `firewalld` but its core backup feature is tied to `iptables`. + +# Dependencies + +- community.general.iptables_state + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `firewall_backup_enabled` | `bool` | `False` | `true` | Flag to enable timestamped backups of `iptables` rules before they are flushed. | +| `firewall_backup_dir` | `path` | `False` | `ansible_user` home directory | Path to the directory where the firewall rule backup files will be saved. | +| `firewall_backup_format` | `str` | `False` | `%Y%m%dT%H%M%S` | Timestamp format string to be used in the backup file names. The format uses standard `strftime` directives. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Disable firewalls and back up iptables rules + ansible.builtin.import_role: + name: cloudera.exe.prereq_firewall + # Backups are enabled by default and saved to the ansible user's home directory. + + - name: Disable firewalls without backing up iptables rules + ansible.builtin.import_role: + name: cloudera.exe.prereq_firewall + vars: + firewall_backup_enabled: false + + - name: Disable firewalls with custom backup directory and format + ansible.builtin.import_role: + name: cloudera.exe.prereq_firewall + vars: + firewall_backup_enabled: true + firewall_backup_dir: "/root/firewall_backups" + firewall_backup_format: "%Y-%m-%d_%H-%M-%S_UTC" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_firewall/defaults/main.yml b/roles/prereq_firewall/defaults/main.yml new file mode 100644 index 00000000..86c8f726 --- /dev/null +++ b/roles/prereq_firewall/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +firewall_backup_enabled: true +firewall_backup_dir: "." +firewall_backup_format: "%Y%m%dT%H%M%S" diff --git a/roles/prereq_firewall/handlers/main.yml b/roles/prereq_firewall/handlers/main.yml new file mode 100644 index 00000000..e2b09427 --- /dev/null +++ b/roles/prereq_firewall/handlers/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# handlers file for prereq_firewall diff --git a/roles/prereq_firewall/meta/argument_specs.yml b/roles/prereq_firewall/meta/argument_specs.yml new file mode 100644 index 00000000..4a04df09 --- /dev/null +++ b/roles/prereq_firewall/meta/argument_specs.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Disable firewalls + description: + - Disable firewalls, if installed, and optionally back up iptable rules. + - Backup files include all iptable tables for both IPv4 and IPv6. + - Backup file naming is C(iptables-rules-[ipv4|ipv6].TIMESTAMP), where TIMESTAMP is UTC. + author: + - Webster Mudge + options: + firewall_backup_enabled: + description: Flag to enable timestamped backups of iptable rules. + type: bool + default: true + firewall_backup_dir: + description: Path of directory of the backup files. Defaults to C(ansible_user) home directory. + type: path + firewall_backup_format: + description: Timestamp format string of the backup files. + type: str + default: "%Y%m%dT%H%M%S" diff --git a/roles/prereq_firewall/molecule/bkup-dir/converge.yml b/roles/prereq_firewall/molecule/bkup-dir/converge.yml new file mode 100644 index 00000000..16a1858d --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/converge.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Disable firewalls and backup iptables + ansible.builtin.import_role: + name: prereq_firewall + vars: + firewall_backup_dir: "/tmp" diff --git a/roles/prereq_firewall/molecule/bkup-dir/create.yml b/roles/prereq_firewall/molecule/bkup-dir/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_firewall/molecule/bkup-dir/destroy.yml b/roles/prereq_firewall/molecule/bkup-dir/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_firewall/molecule/bkup-dir/molecule.yml b/roles/prereq_firewall/molecule/bkup-dir/molecule.yml new file mode 100644 index 00000000..dc217443 --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/molecule.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, TEST_VPC_SUBNET_ID, + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_firewall-rhel9-4 + Project: Molecule testing for prereq_firewall +provisioner: + name: ansible + # inventory: + # group_vars: + # all: + playbooks: + create: ../default/create.yml + destroy: ../default/destroy.yml + converge: converge.yml diff --git a/roles/prereq_firewall/molecule/bkup-dir/requirements.yml b/roles/prereq_firewall/molecule/bkup-dir/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_firewall/molecule/bkup-dir/verify.yml b/roles/prereq_firewall/molecule/bkup-dir/verify.yml new file mode 100644 index 00000000..8b9418d7 --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-dir/verify.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Discover backup files + ansible.builtin.find: + paths: "/tmp" + patterns: + - "iptables-rules-ipv4.*" + - "iptables-rules-ipv6.*" + recurse: false + register: __bkup + failed_when: __bkup.matched != 2 diff --git a/roles/prereq_firewall/molecule/bkup-format/converge.yml b/roles/prereq_firewall/molecule/bkup-format/converge.yml new file mode 100644 index 00000000..6e3052f6 --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/converge.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Disable firewalls and backup iptables + ansible.builtin.import_role: + name: prereq_firewall + vars: + firewall_backup_format: "%Y-TEST" diff --git a/roles/prereq_firewall/molecule/bkup-format/create.yml b/roles/prereq_firewall/molecule/bkup-format/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_firewall/molecule/bkup-format/destroy.yml b/roles/prereq_firewall/molecule/bkup-format/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_firewall/molecule/bkup-format/molecule.yml b/roles/prereq_firewall/molecule/bkup-format/molecule.yml new file mode 100644 index 00000000..dc217443 --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/molecule.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, TEST_VPC_SUBNET_ID, + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_firewall-rhel9-4 + Project: Molecule testing for prereq_firewall +provisioner: + name: ansible + # inventory: + # group_vars: + # all: + playbooks: + create: ../default/create.yml + destroy: ../default/destroy.yml + converge: converge.yml diff --git a/roles/prereq_firewall/molecule/bkup-format/requirements.yml b/roles/prereq_firewall/molecule/bkup-format/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_firewall/molecule/bkup-format/verify.yml b/roles/prereq_firewall/molecule/bkup-format/verify.yml new file mode 100644 index 00000000..a4144602 --- /dev/null +++ b/roles/prereq_firewall/molecule/bkup-format/verify.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Discover backup files + ansible.builtin.find: + paths: "." + patterns: + - "iptables-rules-ipv4.*-TEST" + - "iptables-rules-ipv6.*-TEST" + recurse: false + register: __bkup + failed_when: __bkup.matched != 2 diff --git a/roles/prereq_firewall/molecule/default/converge.yml b/roles/prereq_firewall/molecule/default/converge.yml new file mode 100644 index 00000000..9f8f09ad --- /dev/null +++ b/roles/prereq_firewall/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Disable firewalls and backup iptables + ansible.builtin.import_role: + name: prereq_firewall diff --git a/roles/prereq_firewall/molecule/default/create.yml b/roles/prereq_firewall/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_firewall/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_firewall/molecule/default/destroy.yml b/roles/prereq_firewall/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_firewall/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_firewall/molecule/default/molecule.yml b/roles/prereq_firewall/molecule/default/molecule.yml new file mode 100644 index 00000000..3d6c6ab5 --- /dev/null +++ b/roles/prereq_firewall/molecule/default/molecule.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, TEST_VPC_SUBNET_ID, + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_firewall-rhel9-4 + Project: Molecule testing for prereq_firewall +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_firewall/molecule/default/requirements.yml b/roles/prereq_firewall/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_firewall/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_firewall/molecule/default/verify.yml b/roles/prereq_firewall/molecule/default/verify.yml new file mode 100644 index 00000000..c4c7f3c0 --- /dev/null +++ b/roles/prereq_firewall/molecule/default/verify.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Discover backup files + ansible.builtin.find: + paths: "." + patterns: + - "iptables-rules-ipv4.*" + - "iptables-rules-ipv6.*" + recurse: false + register: __bkup + failed_when: __bkup.matched != 2 diff --git a/roles/prereq_firewall/molecule/no-bkup/converge.yml b/roles/prereq_firewall/molecule/no-bkup/converge.yml new file mode 100644 index 00000000..21914c54 --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/converge.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Disable firewalls and backup iptables + ansible.builtin.import_role: + name: prereq_firewall + vars: + firewall_backup_enabled: false diff --git a/roles/prereq_firewall/molecule/no-bkup/create.yml b/roles/prereq_firewall/molecule/no-bkup/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_firewall/molecule/no-bkup/destroy.yml b/roles/prereq_firewall/molecule/no-bkup/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_firewall/molecule/no-bkup/molecule.yml b/roles/prereq_firewall/molecule/no-bkup/molecule.yml new file mode 100644 index 00000000..dc217443 --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/molecule.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, TEST_VPC_SUBNET_ID, + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_firewall-rhel9-4 + Project: Molecule testing for prereq_firewall +provisioner: + name: ansible + # inventory: + # group_vars: + # all: + playbooks: + create: ../default/create.yml + destroy: ../default/destroy.yml + converge: converge.yml diff --git a/roles/prereq_firewall/molecule/no-bkup/requirements.yml b/roles/prereq_firewall/molecule/no-bkup/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_firewall/molecule/no-bkup/verify.yml b/roles/prereq_firewall/molecule/no-bkup/verify.yml new file mode 100644 index 00000000..638645c8 --- /dev/null +++ b/roles/prereq_firewall/molecule/no-bkup/verify.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Discover backup files + ansible.builtin.find: + paths: "." + patterns: + - "iptables-rules-ipv4.*" + - "iptables-rules-ipv6.*" + recurse: false + register: __bkup + failed_when: __bkup.matched > 0 diff --git a/roles/prereq_firewall/tasks/RedHat-firewall.yml b/roles/prereq_firewall/tasks/RedHat-firewall.yml new file mode 100644 index 00000000..7d203495 --- /dev/null +++ b/roles/prereq_firewall/tasks/RedHat-firewall.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather services + ansible.builtin.service_facts: + +- name: Disable firewall + when: firewall_service in ansible_facts.services + ansible.builtin.service: + name: "{{ firewall_service }}" + enabled: false + state: stopped diff --git a/roles/prereq_firewall/tasks/default-firewall.yml b/roles/prereq_firewall/tasks/default-firewall.yml new file mode 100644 index 00000000..e11188ae --- /dev/null +++ b/roles/prereq_firewall/tasks/default-firewall.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Disable firewall + ansible.builtin.debug: + msg: No instructions found for disabling firewall. Check the distribution type. diff --git a/roles/prereq_firewall/tasks/main.yml b/roles/prereq_firewall/tasks/main.yml new file mode 100644 index 00000000..9a243dff --- /dev/null +++ b/roles/prereq_firewall/tasks/main.yml @@ -0,0 +1,63 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://docs.cloudera.com/cdp-private-cloud-base/7.1.9/installation/topics/cdpdc-disabling-firewall.html + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Back up iptable rules + when: firewall_backup_enabled | bool + block: + - name: Install iptables package + ansible.builtin.package: + name: "{{ iptables_packages }}" + + - name: Get current time in UTC + ansible.builtin.set_fact: + firewall_backup_timestamp: "{{ now(utc=True, fmt=firewall_backup_format) }}" + + - name: Save IPv4 rules + community.general.iptables_state: + path: "{{ firewall_backup_dir }}/iptables-rules-ipv4.{{ firewall_backup_timestamp }}" + ip_version: ipv4 + state: saved + tags: molecule-idempotence-notest + + - name: Save IPv6 rules + community.general.iptables_state: + path: "{{ firewall_backup_dir }}/iptables-rules-ipv6.{{ firewall_backup_timestamp }}" + ip_version: ipv6 + state: saved + tags: molecule-idempotence-notest + +- name: Disable firewall + ansible.builtin.include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}-firewall.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}-firewall.yml" + - "{{ ansible_facts['distribution'] }}-firewall.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}-firewall.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}-firewall.yml" + - "{{ ansible_facts['os_family'] }}-firewall.yml" + - "default-firewall.yml" diff --git a/roles/prereq_firewall/tests/inventory b/roles/prereq_firewall/tests/inventory new file mode 100644 index 00000000..6778d06c --- /dev/null +++ b/roles/prereq_firewall/tests/inventory @@ -0,0 +1,15 @@ +// Copyright 2024 Cloudera, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +localhost diff --git a/roles/prereq_firewall/tests/test.yml b/roles/prereq_firewall/tests/test.yml new file mode 100644 index 00000000..33dc6bdf --- /dev/null +++ b/roles/prereq_firewall/tests/test.yml @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Test + hosts: localhost + remote_user: root + roles: + - prereq_firewall diff --git a/roles/prereq_firewall/vars/RedHat.yml b/roles/prereq_firewall/vars/RedHat.yml new file mode 100644 index 00000000..28781b81 --- /dev/null +++ b/roles/prereq_firewall/vars/RedHat.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +iptables_packages: + - iptables-utils + +firewall_service: firewalld diff --git a/roles/prereq_firewall/vars/default.yml b/roles/prereq_firewall/vars/default.yml new file mode 100644 index 00000000..ef2c3b71 --- /dev/null +++ b/roles/prereq_firewall/vars/default.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +iptables_packages: + - iptables + - ip6tables + +firewall_service: firewalld From 581415493f596540d5402f538749ae702fc3672f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:50 -0400 Subject: [PATCH 12/72] Add Apache Flink prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_flink/README.md | 53 +++ roles/prereq_flink/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_flink/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_flink/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_flink/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_flink/molecule/default/verify.yml | 36 ++ roles/prereq_flink/tasks/main.yml | 39 ++ roles/prereq_flink/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_flink/README.md create mode 100644 roles/prereq_flink/meta/argument_specs.yml create mode 100644 roles/prereq_flink/molecule/default/converge.yml create mode 100644 roles/prereq_flink/molecule/default/create.yml create mode 100644 roles/prereq_flink/molecule/default/destroy.yml create mode 100644 roles/prereq_flink/molecule/default/molecule.yml create mode 100644 roles/prereq_flink/molecule/default/prepare.yml create mode 100644 roles/prereq_flink/molecule/default/requirements.yml create mode 100644 roles/prereq_flink/molecule/default/verify.yml create mode 100644 roles/prereq_flink/tasks/main.yml create mode 100644 roles/prereq_flink/vars/main.yml diff --git a/roles/prereq_flink/README.md b/roles/prereq_flink/README.md new file mode 100644 index 00000000..bed39102 --- /dev/null +++ b/roles/prereq_flink/README.md @@ -0,0 +1,53 @@ +# prereq_flink + +Set up for Flink + +This role prepares a host for Apache Flink usage by creating a dedicated system user and group named `flink`. This user is essential for running Flink processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Flink communication. + +The role will: +- Create the `flink` system user and group. +- Configure home directories and other necessary local paths for the `flink` user, if required. +- Ensure appropriate permissions are set for files and directories related to Flink. +- Configure TLS ACLs to secure Flink communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: flink_nodes + tasks: + - name: Set up the flink user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_flink +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_flink/meta/argument_specs.yml b/roles/prereq_flink/meta/argument_specs.yml new file mode 100644 index 00000000..c1426409 --- /dev/null +++ b/roles/prereq_flink/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Flink + description: | + Set up for Flink usage, notably, create the local C(flink) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_flink/molecule/default/converge.yml b/roles/prereq_flink/molecule/default/converge.yml new file mode 100644 index 00000000..a7c48d27 --- /dev/null +++ b/roles/prereq_flink/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Flink + ansible.builtin.import_role: + name: prereq_flink diff --git a/roles/prereq_flink/molecule/default/create.yml b/roles/prereq_flink/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_flink/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_flink/molecule/default/destroy.yml b/roles/prereq_flink/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_flink/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_flink/molecule/default/molecule.yml b/roles/prereq_flink/molecule/default/molecule.yml new file mode 100644 index 00000000..f189447c --- /dev/null +++ b/roles/prereq_flink/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_flink-rhel9-4 + Project: Molecule testing for prereq_flink +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_flink/molecule/default/prepare.yml b/roles/prereq_flink/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_flink/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_flink/molecule/default/requirements.yml b/roles/prereq_flink/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_flink/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_flink/molecule/default/verify.yml b/roles/prereq_flink/molecule/default/verify.yml new file mode 100644 index 00000000..69210b06 --- /dev/null +++ b/roles/prereq_flink/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for flink users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ flink_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ flink_local_accounts }}" diff --git a/roles/prereq_flink/tasks/main.yml b/roles/prereq_flink/tasks/main.yml new file mode 100644 index 00000000..56d71298 --- /dev/null +++ b/roles/prereq_flink/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ flink_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ flink_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_flink/vars/main.yml b/roles/prereq_flink/vars/main.yml new file mode 100644 index 00000000..70940682 --- /dev/null +++ b/roles/prereq_flink/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +flink_local_accounts: + - user: flink + home: /var/lib/flink + comment: Flink + keystore_acl: true From bc92d663d9e6ed3aadaff2cf47caa0c67905a64a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:51 -0400 Subject: [PATCH 13/72] Add Apache Flume prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_flume/README.md | 53 +++ roles/prereq_flume/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_flume/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_flume/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_flume/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_flume/molecule/default/verify.yml | 36 ++ roles/prereq_flume/tasks/main.yml | 39 ++ roles/prereq_flume/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_flume/README.md create mode 100644 roles/prereq_flume/meta/argument_specs.yml create mode 100644 roles/prereq_flume/molecule/default/converge.yml create mode 100644 roles/prereq_flume/molecule/default/create.yml create mode 100644 roles/prereq_flume/molecule/default/destroy.yml create mode 100644 roles/prereq_flume/molecule/default/molecule.yml create mode 100644 roles/prereq_flume/molecule/default/prepare.yml create mode 100644 roles/prereq_flume/molecule/default/requirements.yml create mode 100644 roles/prereq_flume/molecule/default/verify.yml create mode 100644 roles/prereq_flume/tasks/main.yml create mode 100644 roles/prereq_flume/vars/main.yml diff --git a/roles/prereq_flume/README.md b/roles/prereq_flume/README.md new file mode 100644 index 00000000..8c9e3262 --- /dev/null +++ b/roles/prereq_flume/README.md @@ -0,0 +1,53 @@ +# prereq_flume + +Set up for Flume + +This role prepares a host for Apache Flume usage by creating a dedicated system user and group named `flume`. This user is essential for running Flume processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Flume communication. + +The role will: +- Create the `flume` system user and group. +- Configure home directories and other necessary local paths for the `flume` user, if required. +- Ensure appropriate permissions are set for files and directories related to Flume. +- Configure TLS ACLs to secure Flume communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: flume_nodes + tasks: + - name: Set up the flume user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_flume +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_flume/meta/argument_specs.yml b/roles/prereq_flume/meta/argument_specs.yml new file mode 100644 index 00000000..f8bfe135 --- /dev/null +++ b/roles/prereq_flume/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Flume + description: | + Set up for Flume usage, notably, create the local C(flume) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_flume/molecule/default/converge.yml b/roles/prereq_flume/molecule/default/converge.yml new file mode 100644 index 00000000..e9e17744 --- /dev/null +++ b/roles/prereq_flume/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Flume + ansible.builtin.import_role: + name: prereq_flume diff --git a/roles/prereq_flume/molecule/default/create.yml b/roles/prereq_flume/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_flume/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_flume/molecule/default/destroy.yml b/roles/prereq_flume/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_flume/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_flume/molecule/default/molecule.yml b/roles/prereq_flume/molecule/default/molecule.yml new file mode 100644 index 00000000..2ad04838 --- /dev/null +++ b/roles/prereq_flume/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_flume-rhel9-4 + Project: Molecule testing for prereq_flume +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_flume/molecule/default/prepare.yml b/roles/prereq_flume/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_flume/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_flume/molecule/default/requirements.yml b/roles/prereq_flume/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_flume/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_flume/molecule/default/verify.yml b/roles/prereq_flume/molecule/default/verify.yml new file mode 100644 index 00000000..b304fd0d --- /dev/null +++ b/roles/prereq_flume/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for flume users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ flume_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ flume_local_accounts }}" diff --git a/roles/prereq_flume/tasks/main.yml b/roles/prereq_flume/tasks/main.yml new file mode 100644 index 00000000..ff1604fb --- /dev/null +++ b/roles/prereq_flume/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ flume_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ flume_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_flume/vars/main.yml b/roles/prereq_flume/vars/main.yml new file mode 100644 index 00000000..6415d02f --- /dev/null +++ b/roles/prereq_flume/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +flume_local_accounts: + - user: flume + home: /var/lib/flume-ng + comment: Flume + keystore_acl: true From 22aa462351ff130d0fcabe10d780870aa8b1cf43 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:52 -0400 Subject: [PATCH 14/72] Add Apache Hadoop prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hadoop/README.md | 51 +++ roles/prereq_hadoop/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_hadoop/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ .../prereq_hadoop/molecule/default/verify.yml | 25 ++ roles/prereq_hadoop/tasks/main.yml | 19 + 9 files changed, 696 insertions(+) create mode 100644 roles/prereq_hadoop/README.md create mode 100644 roles/prereq_hadoop/meta/argument_specs.yml create mode 100644 roles/prereq_hadoop/molecule/default/converge.yml create mode 100644 roles/prereq_hadoop/molecule/default/create.yml create mode 100644 roles/prereq_hadoop/molecule/default/destroy.yml create mode 100644 roles/prereq_hadoop/molecule/default/molecule.yml create mode 100644 roles/prereq_hadoop/molecule/default/requirements.yml create mode 100644 roles/prereq_hadoop/molecule/default/verify.yml create mode 100644 roles/prereq_hadoop/tasks/main.yml diff --git a/roles/prereq_hadoop/README.md b/roles/prereq_hadoop/README.md new file mode 100644 index 00000000..15b14f41 --- /dev/null +++ b/roles/prereq_hadoop/README.md @@ -0,0 +1,51 @@ +# prereq_hadoop + +Set up for Hadoop + +This role prepares a host for Apache Hadoop usage by creating a dedicated system group named `hadoop`. This group is essential for managing file system permissions and user access within a Hadoop environment. + +The role will: +- Create the `hadoop` system group. +- Ensure appropriate permissions are set for files and directories that will be used by the Hadoop services and belong to the `hadoop` group. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: hadoop_nodes + tasks: + - name: Set up the hadoop group and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_hadoop +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hadoop/meta/argument_specs.yml b/roles/prereq_hadoop/meta/argument_specs.yml new file mode 100644 index 00000000..ced1a8a7 --- /dev/null +++ b/roles/prereq_hadoop/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Hadoop + description: | + Set up for Hadoop usage, notably, create the local C(hadoop) user group. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_hadoop/molecule/default/converge.yml b/roles/prereq_hadoop/molecule/default/converge.yml new file mode 100644 index 00000000..ed6626ca --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Configure Hadoop + ansible.builtin.import_role: + name: prereq_hadoop diff --git a/roles/prereq_hadoop/molecule/default/create.yml b/roles/prereq_hadoop/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hadoop/molecule/default/destroy.yml b/roles/prereq_hadoop/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hadoop/molecule/default/molecule.yml b/roles/prereq_hadoop/molecule/default/molecule.yml new file mode 100644 index 00000000..58387f2a --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hadoop-rhel9-4 + Project: Molecule testing for prereq_hadoop +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_hadoop/molecule/default/requirements.yml b/roles/prereq_hadoop/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_hadoop/molecule/default/verify.yml b/roles/prereq_hadoop/molecule/default/verify.yml new file mode 100644 index 00000000..010aaa00 --- /dev/null +++ b/roles/prereq_hadoop/molecule/default/verify.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + become: true + tasks: + - name: Check for hadoop group + ansible.builtin.command: grep hadoop /etc/group + register: result + failed_when: result.rc != 0 + changed_when: false diff --git a/roles/prereq_hadoop/tasks/main.yml b/roles/prereq_hadoop/tasks/main.yml new file mode 100644 index 00000000..3cd592f0 --- /dev/null +++ b/roles/prereq_hadoop/tasks/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create hadoop group + ansible.builtin.group: + name: hadoop + state: present From debbde3b41d9c09a1f7e983573931970fd00e45e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:52 -0400 Subject: [PATCH 15/72] Add Apache HBase prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hbase/README.md | 53 +++ roles/prereq_hbase/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_hbase/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_hbase/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_hbase/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_hbase/molecule/default/verify.yml | 32 ++ roles/prereq_hbase/tasks/main.yml | 39 ++ roles/prereq_hbase/vars/main.yml | 20 ++ 11 files changed, 791 insertions(+) create mode 100644 roles/prereq_hbase/README.md create mode 100644 roles/prereq_hbase/meta/argument_specs.yml create mode 100644 roles/prereq_hbase/molecule/default/converge.yml create mode 100644 roles/prereq_hbase/molecule/default/create.yml create mode 100644 roles/prereq_hbase/molecule/default/destroy.yml create mode 100644 roles/prereq_hbase/molecule/default/molecule.yml create mode 100644 roles/prereq_hbase/molecule/default/prepare.yml create mode 100644 roles/prereq_hbase/molecule/default/requirements.yml create mode 100644 roles/prereq_hbase/molecule/default/verify.yml create mode 100644 roles/prereq_hbase/tasks/main.yml create mode 100644 roles/prereq_hbase/vars/main.yml diff --git a/roles/prereq_hbase/README.md b/roles/prereq_hbase/README.md new file mode 100644 index 00000000..c98503f0 --- /dev/null +++ b/roles/prereq_hbase/README.md @@ -0,0 +1,53 @@ +# prereq_hbase + +Set up for Hbase + +This role prepares a host for Apache HBase usage by creating a dedicated system user and group named `hbase`. This user is essential for running HBase processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure HBase communication. + +The role will: +- Create the `hbase` system user and group. +- Configure home directories and other necessary local paths for the `hbase` user, if required. +- Ensure appropriate permissions are set for files and directories related to HBase. +- Configure TLS ACLs to secure HBase communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: hbase_nodes + tasks: + - name: Set up the hbase user and environment + ansible.builtin.import_role: + name: hbase_setup +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hbase/meta/argument_specs.yml b/roles/prereq_hbase/meta/argument_specs.yml new file mode 100644 index 00000000..ddd9bd0b --- /dev/null +++ b/roles/prereq_hbase/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Hbase + description: | + Set up for Hbase usage, notably, create the local C(hbase) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_hbase/molecule/default/converge.yml b/roles/prereq_hbase/molecule/default/converge.yml new file mode 100644 index 00000000..4d84c4bc --- /dev/null +++ b/roles/prereq_hbase/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Hbase + ansible.builtin.import_role: + name: prereq_hbase diff --git a/roles/prereq_hbase/molecule/default/create.yml b/roles/prereq_hbase/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hbase/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hbase/molecule/default/destroy.yml b/roles/prereq_hbase/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hbase/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hbase/molecule/default/molecule.yml b/roles/prereq_hbase/molecule/default/molecule.yml new file mode 100644 index 00000000..39fc6c7e --- /dev/null +++ b/roles/prereq_hbase/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hbase-rhel9-4 + Project: Molecule testing for prereq_hbase +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_hbase/molecule/default/prepare.yml b/roles/prereq_hbase/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_hbase/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_hbase/molecule/default/requirements.yml b/roles/prereq_hbase/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_hbase/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_hbase/molecule/default/verify.yml b/roles/prereq_hbase/molecule/default/verify.yml new file mode 100644 index 00000000..1d05914e --- /dev/null +++ b/roles/prereq_hbase/molecule/default/verify.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for hbase user + ansible.builtin.command: grep hbase /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ hbase_local_accounts }}" diff --git a/roles/prereq_hbase/tasks/main.yml b/roles/prereq_hbase/tasks/main.yml new file mode 100644 index 00000000..7f0a9769 --- /dev/null +++ b/roles/prereq_hbase/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ hbase_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ hbase_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_hbase/vars/main.yml b/roles/prereq_hbase/vars/main.yml new file mode 100644 index 00000000..fd989120 --- /dev/null +++ b/roles/prereq_hbase/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hbase_local_accounts: + - user: hbase + home: /var/lib/hbase + comment: hbase + keystore_acl: true From 301f1ecfaa0254732b6776ac1b33f7c5d3c9dba8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:53 -0400 Subject: [PATCH 16/72] Add HDFS prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hbase/README.md | 2 +- roles/prereq_hdfs/README.md | 53 +++ roles/prereq_hdfs/meta/argument_specs.yml | 23 ++ .../prereq_hdfs/molecule/default/converge.yml | 23 ++ roles/prereq_hdfs/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_hdfs/molecule/default/destroy.yml | 157 ++++++++ .../prereq_hdfs/molecule/default/molecule.yml | 49 +++ .../prereq_hdfs/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_hdfs/molecule/default/verify.yml | 38 ++ roles/prereq_hdfs/tasks/main.yml | 39 ++ roles/prereq_hdfs/vars/main.yml | 20 ++ 12 files changed, 798 insertions(+), 1 deletion(-) create mode 100644 roles/prereq_hdfs/README.md create mode 100644 roles/prereq_hdfs/meta/argument_specs.yml create mode 100644 roles/prereq_hdfs/molecule/default/converge.yml create mode 100644 roles/prereq_hdfs/molecule/default/create.yml create mode 100644 roles/prereq_hdfs/molecule/default/destroy.yml create mode 100644 roles/prereq_hdfs/molecule/default/molecule.yml create mode 100644 roles/prereq_hdfs/molecule/default/prepare.yml create mode 100644 roles/prereq_hdfs/molecule/default/requirements.yml create mode 100644 roles/prereq_hdfs/molecule/default/verify.yml create mode 100644 roles/prereq_hdfs/tasks/main.yml create mode 100644 roles/prereq_hdfs/vars/main.yml diff --git a/roles/prereq_hbase/README.md b/roles/prereq_hbase/README.md index c98503f0..e362e71e 100644 --- a/roles/prereq_hbase/README.md +++ b/roles/prereq_hbase/README.md @@ -31,7 +31,7 @@ None. tasks: - name: Set up the hbase user and environment ansible.builtin.import_role: - name: hbase_setup + name: cloudera.exe.prereq_hbase ``` # License diff --git a/roles/prereq_hdfs/README.md b/roles/prereq_hdfs/README.md new file mode 100644 index 00000000..1cda44e3 --- /dev/null +++ b/roles/prereq_hdfs/README.md @@ -0,0 +1,53 @@ +# prereq_hdfs + +Set up for Hdfs + +This role prepares a host for HDFS (Hadoop Distributed File System) usage by creating a dedicated system user and group named `hdfs`. This user is essential for running HDFS processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure HDFS communication. + +The role will: +- Create the `hdfs` system user and group. +- Configure home directories and other necessary local paths for the `hdfs` user, if required. +- Ensure appropriate permissions are set for files and directories related to HDFS. +- Configure TLS ACLs to secure HDFS communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: hdfs_nodes + tasks: + - name: Set up the hdfs user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_hdfs +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hdfs/meta/argument_specs.yml b/roles/prereq_hdfs/meta/argument_specs.yml new file mode 100644 index 00000000..de674b9f --- /dev/null +++ b/roles/prereq_hdfs/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Hdfs + description: | + Set up for Hdfs usage, notably, create the local C(hdfs) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_hdfs/molecule/default/converge.yml b/roles/prereq_hdfs/molecule/default/converge.yml new file mode 100644 index 00000000..d64df7ce --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Hdfs + ansible.builtin.import_role: + name: prereq_hdfs diff --git a/roles/prereq_hdfs/molecule/default/create.yml b/roles/prereq_hdfs/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hdfs/molecule/default/destroy.yml b/roles/prereq_hdfs/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hdfs/molecule/default/molecule.yml b/roles/prereq_hdfs/molecule/default/molecule.yml new file mode 100644 index 00000000..6607b2ad --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hdfs-rhel9-4 + Project: Molecule testing for prereq_hdfs +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_hdfs/molecule/default/prepare.yml b/roles/prereq_hdfs/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_hdfs/molecule/default/requirements.yml b/roles/prereq_hdfs/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_hdfs/molecule/default/verify.yml b/roles/prereq_hdfs/molecule/default/verify.yml new file mode 100644 index 00000000..c309c8ee --- /dev/null +++ b/roles/prereq_hdfs/molecule/default/verify.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for hadoop group + ansible.builtin.command: grep hadoop /etc/group + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for hdfs user + ansible.builtin.command: grep hdfs /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ hdfs_local_accounts }}" diff --git a/roles/prereq_hdfs/tasks/main.yml b/roles/prereq_hdfs/tasks/main.yml new file mode 100644 index 00000000..dd9fe549 --- /dev/null +++ b/roles/prereq_hdfs/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ hdfs_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ hdfs_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_hdfs/vars/main.yml b/roles/prereq_hdfs/vars/main.yml new file mode 100644 index 00000000..3a9cd216 --- /dev/null +++ b/roles/prereq_hdfs/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hdfs_local_accounts: + - user: hdfs + home: /var/lib/hdfs + comment: hdfs + extra_groups: [hadoop] From a445fd42b381c3a6766d0cf531888bb4a7a2e427 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:54 -0400 Subject: [PATCH 17/72] Add Apache Hive prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hive/README.md | 53 +++ roles/prereq_hive/meta/argument_specs.yml | 23 ++ .../prereq_hive/molecule/default/converge.yml | 23 ++ roles/prereq_hive/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_hive/molecule/default/destroy.yml | 157 ++++++++ .../prereq_hive/molecule/default/molecule.yml | 49 +++ .../prereq_hive/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_hive/molecule/default/verify.yml | 24 ++ roles/prereq_hive/tasks/main.yml | 39 ++ roles/prereq_hive/vars/main.yml | 20 ++ 11 files changed, 783 insertions(+) create mode 100644 roles/prereq_hive/README.md create mode 100644 roles/prereq_hive/meta/argument_specs.yml create mode 100644 roles/prereq_hive/molecule/default/converge.yml create mode 100644 roles/prereq_hive/molecule/default/create.yml create mode 100644 roles/prereq_hive/molecule/default/destroy.yml create mode 100644 roles/prereq_hive/molecule/default/molecule.yml create mode 100644 roles/prereq_hive/molecule/default/prepare.yml create mode 100644 roles/prereq_hive/molecule/default/requirements.yml create mode 100644 roles/prereq_hive/molecule/default/verify.yml create mode 100644 roles/prereq_hive/tasks/main.yml create mode 100644 roles/prereq_hive/vars/main.yml diff --git a/roles/prereq_hive/README.md b/roles/prereq_hive/README.md new file mode 100644 index 00000000..69668604 --- /dev/null +++ b/roles/prereq_hive/README.md @@ -0,0 +1,53 @@ +# prereq_hive + +Set up for Hive + +This role prepares a host for Apache Hive usage by creating a dedicated system user and group named `hive`. This user is essential for running Hive processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Hive communication. + +The role will: +- Create the `hive` system user and group. +- Configure home directories and other necessary local paths for the `hive` user, if required. +- Ensure appropriate permissions are set for files and directories related to Hive. +- Configure TLS ACLs to secure Hive communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: hive_nodes + tasks: + - name: Set up the hive user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_hive +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hive/meta/argument_specs.yml b/roles/prereq_hive/meta/argument_specs.yml new file mode 100644 index 00000000..29adc96b --- /dev/null +++ b/roles/prereq_hive/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Hive + description: | + Set up for Hive usage, notably, create the local C(hive) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_hive/molecule/default/converge.yml b/roles/prereq_hive/molecule/default/converge.yml new file mode 100644 index 00000000..9d54246a --- /dev/null +++ b/roles/prereq_hive/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Hive + ansible.builtin.import_role: + name: prereq_hive diff --git a/roles/prereq_hive/molecule/default/create.yml b/roles/prereq_hive/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hive/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hive/molecule/default/destroy.yml b/roles/prereq_hive/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hive/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hive/molecule/default/molecule.yml b/roles/prereq_hive/molecule/default/molecule.yml new file mode 100644 index 00000000..4be3d3f6 --- /dev/null +++ b/roles/prereq_hive/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hive-rhel9-4 + Project: Molecule testing for prereq_hive +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_hive/molecule/default/prepare.yml b/roles/prereq_hive/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_hive/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_hive/molecule/default/requirements.yml b/roles/prereq_hive/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_hive/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_hive/molecule/default/verify.yml b/roles/prereq_hive/molecule/default/verify.yml new file mode 100644 index 00000000..6aef2032 --- /dev/null +++ b/roles/prereq_hive/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check for hive user + ansible.builtin.command: grep hive /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false diff --git a/roles/prereq_hive/tasks/main.yml b/roles/prereq_hive/tasks/main.yml new file mode 100644 index 00000000..e0145668 --- /dev/null +++ b/roles/prereq_hive/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ hive_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ hive_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_hive/vars/main.yml b/roles/prereq_hive/vars/main.yml new file mode 100644 index 00000000..4d76e2f4 --- /dev/null +++ b/roles/prereq_hive/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hive_local_accounts: + - user: hive + home: /var/lib/hive + comment: Hive + keystore_acl: true From c8353933290e62270e1ff490388b6158a3986da0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:54 -0400 Subject: [PATCH 18/72] Add Apache Hive database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hive_database/README.md | 79 ++++ roles/prereq_hive_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 32 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_hive_database/tasks/main.yml | 24 ++ 11 files changed, 852 insertions(+) create mode 100644 roles/prereq_hive_database/README.md create mode 100644 roles/prereq_hive_database/defaults/main.yml create mode 100644 roles/prereq_hive_database/meta/argument_specs.yml create mode 100644 roles/prereq_hive_database/molecule/default/converge.yml create mode 100644 roles/prereq_hive_database/molecule/default/create.yml create mode 100644 roles/prereq_hive_database/molecule/default/destroy.yml create mode 100644 roles/prereq_hive_database/molecule/default/molecule.yml create mode 100644 roles/prereq_hive_database/molecule/default/prepare.yml create mode 100644 roles/prereq_hive_database/molecule/default/requirements.yml create mode 100644 roles/prereq_hive_database/molecule/default/verify.yml create mode 100644 roles/prereq_hive_database/tasks/main.yml diff --git a/roles/prereq_hive_database/README.md b/roles/prereq_hive_database/README.md new file mode 100644 index 00000000..0ea99fe8 --- /dev/null +++ b/roles/prereq_hive_database/README.md @@ -0,0 +1,79 @@ +# prereq_hive_database + +Set up database and user accounts for Hive + +This role automates the setup of a database and its associated user accounts specifically for Hive services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `hive_database`. +- Create a new database user specified by `hive_username` with the password from `hive_password`. +- Grant ownership and all necessary privileges to the `hive_username` for the new database. +- Ensure the database is configured correctly for Hive operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `hive_username` | `str` | `False` | `hive` | The username for the Hive database user. This user will also be the owner of the database. | +| `hive_password` | `str` | `False` | `hive` | The password for the Hive database user. It is highly recommended to override this default in production. | +| `hive_database` | `str` | `False` | `hive` | The name of the database to be created for Hive. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Hive database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_hive_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Hive database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_hive_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + hive_username: "my_hive_user" + hive_password: "a_strong_hive_password" + hive_database: "my_hive_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hive_database/defaults/main.yml b/roles/prereq_hive_database/defaults/main.yml new file mode 100644 index 00000000..a69e0763 --- /dev/null +++ b/roles/prereq_hive_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +hive_username: hive +hive_password: hive +hive_database: hive diff --git a/roles/prereq_hive_database/meta/argument_specs.yml b/roles/prereq_hive_database/meta/argument_specs.yml new file mode 100644 index 00000000..1fdb00f3 --- /dev/null +++ b/roles/prereq_hive_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Hive + description: + - Set up the Hive database and its associated user accounts, ensuring proper configuration for Hive operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `hive_username`, `hive_password`, and + `hive_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + hive_username: + description: The username for the Hive database user and owner of the database. + type: str + required: false + default: hive + hive_password: + description: The password for the Hive database user. + type: str + required: false + default: hive + hive_database: + description: The name of the database to be created for Hive. + type: str + required: false + default: hive diff --git a/roles/prereq_hive_database/molecule/default/converge.yml b/roles/prereq_hive_database/molecule/default/converge.yml new file mode 100644 index 00000000..6a31ce08 --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Hive database and configure its associated user account. + ansible.builtin.import_role: + name: prereq_hive_database diff --git a/roles/prereq_hive_database/molecule/default/create.yml b/roles/prereq_hive_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hive_database/molecule/default/destroy.yml b/roles/prereq_hive_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hive_database/molecule/default/molecule.yml b/roles/prereq_hive_database/molecule/default/molecule.yml new file mode 100644 index 00000000..937659f0 --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hive_database-rhel9-4 + Project: Molecule testing for prereq_hive_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_hive_database/molecule/default/prepare.yml b/roles/prereq_hive_database/molecule/default/prepare.yml new file mode 100644 index 00000000..2dcd1235 --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/prepare.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres + when: database_type == 'postgresql' diff --git a/roles/prereq_hive_database/molecule/default/requirements.yml b/roles/prereq_hive_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_hive_database/molecule/default/verify.yml b/roles/prereq_hive_database/molecule/default/verify.yml new file mode 100644 index 00000000..a9a4ebb7 --- /dev/null +++ b/roles/prereq_hive_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'hive';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database hive does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'hive';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User hive does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_hive_database/tasks/main.yml b/roles/prereq_hive_database/tasks/main.yml new file mode 100644 index 00000000..08422045 --- /dev/null +++ b/roles/prereq_hive_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ hive_details }}" + hive_details: + - user: "{{ hive_username }}" + password: "{{ hive_password }}" + db: "{{ hive_database }}" + no_log: true From b07bbd5037cf6c8158059be10b43fb65e6bff670 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:55 -0400 Subject: [PATCH 19/72] Add HttpFS prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_httpfs/README.md | 53 +++ roles/prereq_httpfs/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_httpfs/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_httpfs/molecule/default/verify.yml | 36 ++ roles/prereq_httpfs/tasks/main.yml | 39 ++ roles/prereq_httpfs/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_httpfs/README.md create mode 100644 roles/prereq_httpfs/meta/argument_specs.yml create mode 100644 roles/prereq_httpfs/molecule/default/converge.yml create mode 100644 roles/prereq_httpfs/molecule/default/create.yml create mode 100644 roles/prereq_httpfs/molecule/default/destroy.yml create mode 100644 roles/prereq_httpfs/molecule/default/molecule.yml create mode 100644 roles/prereq_httpfs/molecule/default/prepare.yml create mode 100644 roles/prereq_httpfs/molecule/default/requirements.yml create mode 100644 roles/prereq_httpfs/molecule/default/verify.yml create mode 100644 roles/prereq_httpfs/tasks/main.yml create mode 100644 roles/prereq_httpfs/vars/main.yml diff --git a/roles/prereq_httpfs/README.md b/roles/prereq_httpfs/README.md new file mode 100644 index 00000000..505b44ff --- /dev/null +++ b/roles/prereq_httpfs/README.md @@ -0,0 +1,53 @@ +# prereq_httpfs + +Set up for HttpFS + +This role prepares a host for HttpFS usage by creating a dedicated system user and group named `httpfs`. This user is essential for running Httpfs processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Httpfs communication. + +The role will: +- Create the `httpfs` system user and group. +- Configure home directories and other necessary local paths for the `httpfs` user, if required. +- Ensure appropriate permissions are set for files and directories related to Httpfs. +- Configure TLS ACLs to secure Httpfs communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: httpfs_nodes + tasks: + - name: Set up the httpfs user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_httpfs +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_httpfs/meta/argument_specs.yml b/roles/prereq_httpfs/meta/argument_specs.yml new file mode 100644 index 00000000..d8bafcb1 --- /dev/null +++ b/roles/prereq_httpfs/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for HttpFS + description: | + Set up for HttpFS usage, notably, create the local C(httpfs) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_httpfs/molecule/default/converge.yml b/roles/prereq_httpfs/molecule/default/converge.yml new file mode 100644 index 00000000..7bed2dfb --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Httpfs + ansible.builtin.import_role: + name: prereq_httpfs diff --git a/roles/prereq_httpfs/molecule/default/create.yml b/roles/prereq_httpfs/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_httpfs/molecule/default/destroy.yml b/roles/prereq_httpfs/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_httpfs/molecule/default/molecule.yml b/roles/prereq_httpfs/molecule/default/molecule.yml new file mode 100644 index 00000000..707519ae --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_httpfs-rhel9-4 + Project: Molecule testing for prereq_httpfs +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_httpfs/molecule/default/prepare.yml b/roles/prereq_httpfs/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_httpfs/molecule/default/requirements.yml b/roles/prereq_httpfs/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_httpfs/molecule/default/verify.yml b/roles/prereq_httpfs/molecule/default/verify.yml new file mode 100644 index 00000000..e65f8591 --- /dev/null +++ b/roles/prereq_httpfs/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for httpfs users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ httpfs_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ httpfs_local_accounts }}" diff --git a/roles/prereq_httpfs/tasks/main.yml b/roles/prereq_httpfs/tasks/main.yml new file mode 100644 index 00000000..2b6e2926 --- /dev/null +++ b/roles/prereq_httpfs/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ httpfs_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ httpfs_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_httpfs/vars/main.yml b/roles/prereq_httpfs/vars/main.yml new file mode 100644 index 00000000..b122aba0 --- /dev/null +++ b/roles/prereq_httpfs/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +httpfs_local_accounts: + - user: httpfs + home: /var/lib/hadoop-httpfs + comment: Hadoop HTTPFS + keystore_acl: true From 77a3cdacd54eab48c147bd3edc96c295542ada19 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:56 -0400 Subject: [PATCH 20/72] Add Hue prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hue/README.md | 61 ++++ roles/prereq_hue/defaults/main.yml | 16 + roles/prereq_hue/meta/argument_specs.yml | 29 ++ .../prereq_hue/molecule/default/converge.yml | 23 ++ roles/prereq_hue/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_hue/molecule/default/destroy.yml | 157 ++++++++ .../prereq_hue/molecule/default/molecule.yml | 49 +++ roles/prereq_hue/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_hue/molecule/default/verify.yml | 36 ++ roles/prereq_hue/tasks/main.yml | 61 ++++ roles/prereq_hue/vars/main.yml | 21 ++ 12 files changed, 848 insertions(+) create mode 100644 roles/prereq_hue/README.md create mode 100644 roles/prereq_hue/defaults/main.yml create mode 100644 roles/prereq_hue/meta/argument_specs.yml create mode 100644 roles/prereq_hue/molecule/default/converge.yml create mode 100644 roles/prereq_hue/molecule/default/create.yml create mode 100644 roles/prereq_hue/molecule/default/destroy.yml create mode 100644 roles/prereq_hue/molecule/default/molecule.yml create mode 100644 roles/prereq_hue/molecule/default/prepare.yml create mode 100644 roles/prereq_hue/molecule/default/requirements.yml create mode 100644 roles/prereq_hue/molecule/default/verify.yml create mode 100644 roles/prereq_hue/tasks/main.yml create mode 100644 roles/prereq_hue/vars/main.yml diff --git a/roles/prereq_hue/README.md b/roles/prereq_hue/README.md new file mode 100644 index 00000000..268abad7 --- /dev/null +++ b/roles/prereq_hue/README.md @@ -0,0 +1,61 @@ +# prereq_hue + +Set up for Hue + +This role prepares a host for Hue usage by creating a dedicated system user and group named `hue`. This user is essential for running Hue processes with appropriate permissions and isolation. The role also handles configuration for Hue's Kerberos and TLS requirements, including setting up ACLs on TLS entities and updating Kerberos encryption types. + +The role will: +- Create the `hue` system user and group. +- Configure home directories and other necessary local paths for the `hue` user, if required. +- Ensure appropriate permissions are set for files and directories related to Hue. +- Optionally, set up TLS ACLs to secure Hue communication, if needed. +- Optionally, update Kerberos encryption types as required for Hue's secure operations. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and system files. +- The Kerberos configuration file at `kerberos_config_path` must exist on the target host or be managed by another role. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `kerberos_config_path` | `path` | `False` | `/etc/krb5.conf` | Path to the Kerberos configuration file on the target host. This file will be configured for Hue's Kerberos settings. | + +# Example Playbook + +```yaml +- hosts: hue_nodes + tasks: + - name: Set up the hue user and default Kerberos configuration + ansible.builtin.import_role: + name: cloudera.exe.prereq_hue + + - name: Set up the hue user with a custom Kerberos configuration path + ansible.builtin.import_role: + name: cloudera.exe.prereq_hue + vars: + kerberos_config_path: "/etc/custom/krb5.conf" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hue/defaults/main.yml b/roles/prereq_hue/defaults/main.yml new file mode 100644 index 00000000..e07d9fb9 --- /dev/null +++ b/roles/prereq_hue/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kerberos_config_path: "/etc/krb5.conf" diff --git a/roles/prereq_hue/meta/argument_specs.yml b/roles/prereq_hue/meta/argument_specs.yml new file mode 100644 index 00000000..7fb4cdde --- /dev/null +++ b/roles/prereq_hue/meta/argument_specs.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Hue + description: | + Set up for Hue usage, notably, create the local C(hue) user. + Optionally, set up ACLs on TLS entities. + Optionally, update Kerberos encryption types. + author: Cloudera Labs + options: + kerberos_config_path: + description: + - Path to the Kerberos configuration file. + type: path + default: "/etc/krb5.conf" diff --git a/roles/prereq_hue/molecule/default/converge.yml b/roles/prereq_hue/molecule/default/converge.yml new file mode 100644 index 00000000..037e0edb --- /dev/null +++ b/roles/prereq_hue/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Hue + ansible.builtin.import_role: + name: prereq_hue diff --git a/roles/prereq_hue/molecule/default/create.yml b/roles/prereq_hue/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hue/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hue/molecule/default/destroy.yml b/roles/prereq_hue/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hue/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hue/molecule/default/molecule.yml b/roles/prereq_hue/molecule/default/molecule.yml new file mode 100644 index 00000000..e03bce4c --- /dev/null +++ b/roles/prereq_hue/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hue-rhel9-4 + Project: Molecule testing for prereq_hue +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_hue/molecule/default/prepare.yml b/roles/prereq_hue/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_hue/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_hue/molecule/default/requirements.yml b/roles/prereq_hue/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_hue/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_hue/molecule/default/verify.yml b/roles/prereq_hue/molecule/default/verify.yml new file mode 100644 index 00000000..933dc158 --- /dev/null +++ b/roles/prereq_hue/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for hue users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ hue_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ hue_local_accounts }}" diff --git a/roles/prereq_hue/tasks/main.yml b/roles/prereq_hue/tasks/main.yml new file mode 100644 index 00000000..6264c312 --- /dev/null +++ b/roles/prereq_hue/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ hue_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ hue_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl + +- name: Get details of Kerberos configuration + ansible.builtin.stat: + path: "{{ kerberos_config_path }}" + register: __krb + +- name: Update Kerberos encryption types + when: __krb.stat.exists + community.general.ini_file: + path: "{{ kerberos_config_path }}" + section: libdefaults + option: "{{ __hue_krb.key }}" + value: "{{ __hue_krb.value }}" + mode: "0755" + loop: "{{ entries | dict2items }}" + loop_control: + loop_var: __hue_krb + label: "{{ __hue_krb.key }}" + vars: + entries: + default_tgs_enctypes: des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 + default_tkt_enctypes: des3-cbc-sha1 aes256-cts-hmac-sha1-96 arcfour-hmac aes128-cts-hmac-sha1-96 des-cbc-md5 diff --git a/roles/prereq_hue/vars/main.yml b/roles/prereq_hue/vars/main.yml new file mode 100644 index 00000000..26fd053b --- /dev/null +++ b/roles/prereq_hue/vars/main.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hue_local_accounts: + - user: hue + home: /var/lib/hue + comment: hue + key_acl: true + key_password_acl: true From f7c50d7fdf60ac8c5c6a11eb4e580196b7df311d Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:57 -0400 Subject: [PATCH 21/72] Add Hue database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_hue_database/README.md | 79 ++++ roles/prereq_hue_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_hue_database/tasks/main.yml | 24 ++ 11 files changed, 853 insertions(+) create mode 100644 roles/prereq_hue_database/README.md create mode 100644 roles/prereq_hue_database/defaults/main.yml create mode 100644 roles/prereq_hue_database/meta/argument_specs.yml create mode 100644 roles/prereq_hue_database/molecule/default/converge.yml create mode 100644 roles/prereq_hue_database/molecule/default/create.yml create mode 100644 roles/prereq_hue_database/molecule/default/destroy.yml create mode 100644 roles/prereq_hue_database/molecule/default/molecule.yml create mode 100644 roles/prereq_hue_database/molecule/default/prepare.yml create mode 100644 roles/prereq_hue_database/molecule/default/requirements.yml create mode 100644 roles/prereq_hue_database/molecule/default/verify.yml create mode 100644 roles/prereq_hue_database/tasks/main.yml diff --git a/roles/prereq_hue_database/README.md b/roles/prereq_hue_database/README.md new file mode 100644 index 00000000..1731dcae --- /dev/null +++ b/roles/prereq_hue_database/README.md @@ -0,0 +1,79 @@ +# prereq_hue_database + +Set up database and user accounts for Hue + +This role automates the setup of a database and its associated user accounts specifically for Hue services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `hue_database`. +- Create a new database user specified by `hue_username` with the password from `hue_password`. +- Grant ownership and all necessary privileges to the `hue_username` for the new database. +- Ensure the database is configured correctly for Hue operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `hue_username` | `str` | `False` | `hue` | The username for the Hue database user. This user will also be the owner of the database. | +| `hue_password` | `str` | `False` | `hue` | The password for the Hue database user. It is highly recommended to override this default in production. | +| `hue_database` | `str` | `False` | `hue` | The name of the database to be created for Hue. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Hue database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_hue_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Hue database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_hue_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + hue_username: "my_hue_user" + hue_password: "a_strong_hue_password" + hue_database: "my_hue_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_hue_database/defaults/main.yml b/roles/prereq_hue_database/defaults/main.yml new file mode 100644 index 00000000..2ad0d882 --- /dev/null +++ b/roles/prereq_hue_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +hue_username: hue +hue_password: hue +hue_database: hue diff --git a/roles/prereq_hue_database/meta/argument_specs.yml b/roles/prereq_hue_database/meta/argument_specs.yml new file mode 100644 index 00000000..a90a8304 --- /dev/null +++ b/roles/prereq_hue_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Hue + description: + - Set up the Hue database and its associated user accounts, ensuring proper configuration for Hue operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `hue_username`, `hue_password`, and + `hue_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + hue_username: + description: The username for the Hue database user and owner of the database. + type: str + required: false + default: hue + hue_password: + description: The password for the Hue database user. + type: str + required: false + default: hue + hue_database: + description: The name of the database to be created for Hue. + type: str + required: false + default: hue diff --git a/roles/prereq_hue_database/molecule/default/converge.yml b/roles/prereq_hue_database/molecule/default/converge.yml new file mode 100644 index 00000000..c706e771 --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Hue database and configure its associated user account. + ansible.builtin.import_role: + name: prereq_hue_database diff --git a/roles/prereq_hue_database/molecule/default/create.yml b/roles/prereq_hue_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_hue_database/molecule/default/destroy.yml b/roles/prereq_hue_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_hue_database/molecule/default/molecule.yml b/roles/prereq_hue_database/molecule/default/molecule.yml new file mode 100644 index 00000000..423fdd1d --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_hue_database-rhel9-4 + Project: Molecule testing for prereq_hue_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_hue_database/molecule/default/prepare.yml b/roles/prereq_hue_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_hue_database/molecule/default/requirements.yml b/roles/prereq_hue_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_hue_database/molecule/default/verify.yml b/roles/prereq_hue_database/molecule/default/verify.yml new file mode 100644 index 00000000..4fa53cb8 --- /dev/null +++ b/roles/prereq_hue_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'hue';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database hue does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'hue';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User hue does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_hue_database/tasks/main.yml b/roles/prereq_hue_database/tasks/main.yml new file mode 100644 index 00000000..0275615a --- /dev/null +++ b/roles/prereq_hue_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ hue_details }}" + hue_details: + - user: "{{ hue_username }}" + password: "{{ hue_password }}" + db: "{{ hue_database }}" + no_log: true From 1a55847788cffb78d6724cd176136ce17e2ee20a Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:57 -0400 Subject: [PATCH 22/72] Add Apache Impala prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_impala/README.md | 71 ++++ roles/prereq_impala/meta/argument_specs.yml | 51 +++ .../molecule/default/converge.yml | 23 ++ .../prereq_impala/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_impala/molecule/default/verify.yml | 38 ++ roles/prereq_impala/tasks/main.yml | 39 ++ roles/prereq_impala/vars/main.yml | 22 ++ 11 files changed, 845 insertions(+) create mode 100644 roles/prereq_impala/README.md create mode 100644 roles/prereq_impala/meta/argument_specs.yml create mode 100644 roles/prereq_impala/molecule/default/converge.yml create mode 100644 roles/prereq_impala/molecule/default/create.yml create mode 100644 roles/prereq_impala/molecule/default/destroy.yml create mode 100644 roles/prereq_impala/molecule/default/molecule.yml create mode 100644 roles/prereq_impala/molecule/default/prepare.yml create mode 100644 roles/prereq_impala/molecule/default/requirements.yml create mode 100644 roles/prereq_impala/molecule/default/verify.yml create mode 100644 roles/prereq_impala/tasks/main.yml create mode 100644 roles/prereq_impala/vars/main.yml diff --git a/roles/prereq_impala/README.md b/roles/prereq_impala/README.md new file mode 100644 index 00000000..6e30887e --- /dev/null +++ b/roles/prereq_impala/README.md @@ -0,0 +1,71 @@ +# prereq_impala + +Set up for Impala + +This role prepares a host for Apache Impala usage by creating a dedicated system user and group named `impala`. This user is essential for running Impala processes with appropriate permissions and isolation. The role can also optionally configure TLS for Impala communication, including managing keystore, private key, and password files, and setting up file system permissions for these entities. + +The role will: +- Create the `impala` system user and group. +- Configure home directories and other necessary local paths for the `impala` user, if required. +- Ensure appropriate permissions are set for files and directories related to Impala. +- If TLS is enabled via the provided parameters, the role will: + - Set up Access Control Lists (ACLs) on TLS-related files. + - Manage hardlinks for TLS files as needed for generic paths. + - Ensure TLS key password files are securely configured. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. +- If any TLS path parameters are specified, the corresponding certificate, key, and password files must exist on the target host or be managed by another role. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `tls_keystore_path` | `path` | `False` | | Path to the TLS keystore file. | +| `tls_keystore_path_generic` | `path` | `False` | | Path to a hardlink that points to the TLS keystore. Used to provide a consistent, generic path. | +| `tls_key_path` | `path` | `False` | | Path to the encrypted TLS private key file. | +| `tls_key_path_generic` | `path` | `False` | | Path to a hardlink that points to the encrypted TLS private key. | +| `tls_key_password_file` | `path` | `False` | | Path to the file containing the password for the TLS private key. | +| `tls_key_path_plaintext` | `path` | `False` | | Path to the unencrypted TLS private key file. | +| `tls_key_path_plaintext_generic` | `path` | `False` | | Path to a hardlink that points to the unencrypted TLS private key. | + +# Example Playbook + +```yaml +- hosts: impala_nodes + tasks: + - name: Set up the impala user and environment without TLS + ansible.builtin.import_role: + name: cloudera.exe.prereq_impala + + - name: Set up impala with TLS configuration + ansible.builtin.import_role: + name: cloudera.exe.prereq_impala + vars: + tls_keystore_path: "/opt/certs/impala/keystore.jks" + tls_key_path: "/opt/certs/impala/impala.key" + tls_key_password_file: "/opt/certs/impala/password.txt" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_impala/meta/argument_specs.yml b/roles/prereq_impala/meta/argument_specs.yml new file mode 100644 index 00000000..3eba037d --- /dev/null +++ b/roles/prereq_impala/meta/argument_specs.yml @@ -0,0 +1,51 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Impala + description: | + Set up for Impala usage, notably, create the local C(impala) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: + tls_keystore_path: + description: + - Path of the TLS keystore. + type: path + tls_keystore_path_generic: + description: + - Path of the hardlink to the TLS keystore. + type: path + tls_key_path: + description: + - Path of the encrypted TLS private key. + type: path + tls_key_path_generic: + description: + - Path of the hardlink to the encrypted TLS private key. + type: path + tls_key_password_file: + description: + - Path of the TLS private key password file. + type: path + tls_key_path_plaintext: + description: + - Path of the unencrypted TLS private key. + type: path + tls_key_path_plaintext_generic: + description: + - Path of the hardlink to the unencrypted TLS private key. + type: path diff --git a/roles/prereq_impala/molecule/default/converge.yml b/roles/prereq_impala/molecule/default/converge.yml new file mode 100644 index 00000000..3e30f911 --- /dev/null +++ b/roles/prereq_impala/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Impala + ansible.builtin.import_role: + name: prereq_impala diff --git a/roles/prereq_impala/molecule/default/create.yml b/roles/prereq_impala/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_impala/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_impala/molecule/default/destroy.yml b/roles/prereq_impala/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_impala/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_impala/molecule/default/molecule.yml b/roles/prereq_impala/molecule/default/molecule.yml new file mode 100644 index 00000000..501bc5e5 --- /dev/null +++ b/roles/prereq_impala/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_impala-rhel9-4 + Project: Molecule testing for prereq_impala +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_impala/molecule/default/prepare.yml b/roles/prereq_impala/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_impala/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_impala/molecule/default/requirements.yml b/roles/prereq_impala/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_impala/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_impala/molecule/default/verify.yml b/roles/prereq_impala/molecule/default/verify.yml new file mode 100644 index 00000000..caf45f12 --- /dev/null +++ b/roles/prereq_impala/molecule/default/verify.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for impala user + ansible.builtin.command: grep impala /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for hive user group membership + ansible.builtin.command: groups impala + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("hive") + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ impala_local_accounts }}" diff --git a/roles/prereq_impala/tasks/main.yml b/roles/prereq_impala/tasks/main.yml new file mode 100644 index 00000000..a050744d --- /dev/null +++ b/roles/prereq_impala/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ impala_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ impala_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_impala/vars/main.yml b/roles/prereq_impala/vars/main.yml new file mode 100644 index 00000000..8d1768e9 --- /dev/null +++ b/roles/prereq_impala/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +impala_local_accounts: + - user: impala + home: /var/lib/impala + comment: Impala + extra_groups: [hive] + key_acl: true + key_password_acl: true From ae65848100c481b60ac14c31f96ab619e90074f1 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:58 -0400 Subject: [PATCH 23/72] Add JDK prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_jdk/README.md | 78 ++++ roles/prereq_jdk/defaults/main.yml | 21 ++ roles/prereq_jdk/library/jdk_facts.py | 158 ++++++++ roles/prereq_jdk/meta/argument_specs.yml | 61 ++++ .../prereq_jdk/molecule/default/converge.yml | 23 ++ roles/prereq_jdk/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_jdk/molecule/default/destroy.yml | 157 ++++++++ .../prereq_jdk/molecule/default/molecule.yml | 39 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_jdk/molecule/default/verify.yml | 19 + .../molecule/openjdk-8/converge.yml | 25 ++ .../molecule/openjdk-8/molecule.yml | 42 +++ .../molecule/openjdk-8/requirements.yml | 21 ++ .../prereq_jdk/molecule/openjdk-8/verify.yml | 19 + roles/prereq_jdk/tasks/main.yml | 108 ++++++ roles/prereq_jdk/tasks/openjdk_tasks.yml | 20 ++ roles/prereq_jdk/vars/RedHat.yml | 27 ++ roles/prereq_jdk/vars/default.yml | 13 + roles/prereq_jdk/vars/main.yml | 13 + 19 files changed, 1201 insertions(+) create mode 100644 roles/prereq_jdk/README.md create mode 100644 roles/prereq_jdk/defaults/main.yml create mode 100644 roles/prereq_jdk/library/jdk_facts.py create mode 100644 roles/prereq_jdk/meta/argument_specs.yml create mode 100644 roles/prereq_jdk/molecule/default/converge.yml create mode 100644 roles/prereq_jdk/molecule/default/create.yml create mode 100644 roles/prereq_jdk/molecule/default/destroy.yml create mode 100644 roles/prereq_jdk/molecule/default/molecule.yml create mode 100644 roles/prereq_jdk/molecule/default/requirements.yml create mode 100644 roles/prereq_jdk/molecule/default/verify.yml create mode 100644 roles/prereq_jdk/molecule/openjdk-8/converge.yml create mode 100644 roles/prereq_jdk/molecule/openjdk-8/molecule.yml create mode 100644 roles/prereq_jdk/molecule/openjdk-8/requirements.yml create mode 100644 roles/prereq_jdk/molecule/openjdk-8/verify.yml create mode 100644 roles/prereq_jdk/tasks/main.yml create mode 100644 roles/prereq_jdk/tasks/openjdk_tasks.yml create mode 100644 roles/prereq_jdk/vars/RedHat.yml create mode 100644 roles/prereq_jdk/vars/default.yml create mode 100644 roles/prereq_jdk/vars/main.yml diff --git a/roles/prereq_jdk/README.md b/roles/prereq_jdk/README.md new file mode 100644 index 00000000..8832d38a --- /dev/null +++ b/roles/prereq_jdk/README.md @@ -0,0 +1,78 @@ +# prereq_jdk + +Set up JDK + +This role automates the setup of a Java Development Kit (JDK) on a host. It can optionally install the JDK packages from various providers (OpenJDK, Oracle, Azul), handle version management, and perform post-installation configuration. For older JDK versions (9 and below), it can also enable the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy to support stronger encryption. + +The role will: +- Install the specified JDK packages if `jdk_install_packages` is `true`. +- For JDK versions 8 and below, it will apply the JCE Unlimited Strength Jurisdiction Policy if needed, by modifying `java.security` files. +- If multiple `java.security` files are found during JCE configuration, it will either proceed or halt based on the `jdk_security_paths_override` flag. +- For JDKs installed from Cloudera's repository, the role will ensure that any missing symbolic links are created to support a consistent JDK installation path. + +# Requirements + +- Root or `sudo` privileges are required to install packages and modify system-wide configuration files. +- Network access to the package repositories for the chosen JDK provider. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `jdk_provider` | `str` | `False` | `openjdk` | The JDK vendor or provider to use for installation. Valid choices are `openjdk`, `oracle`, and `azul`. | +| `jdk_install_packages` | `bool` | `False` | `True` | Flag to enable or disable the installation of JDK packages. If `false`, the role will assume a JDK is already installed and will only perform configuration tasks. | +| `jdk_packages` | `list` of `str` | `False` | - | A list of OS packages to install if `jdk_install_packages` is `true`. If not specified, the role will use default package names based on `jdk_provider` and `jdk_version`. | +| `jdk_version` | `int` | `False` | `17` | The supported JDK version to install. Valid choices are `8`, `11`, and `17`. | +| `jdk_security_paths` | `list` of `path` | `False` | - | A list of paths to search for `java.security` files. The role will only apply JCE changes to files in these locations. | +| `jdk_security_paths_override` | `bool` | `False` | `False` | Flag to control behavior when multiple `java.security` files are found in the specified paths. If `true`, the role will continue with JCE changes even if multiple files are found. If `false`, the role will fail, requiring a more specific path list. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Set up default OpenJDK 17 installation + ansible.builtin.import_role: + name: cloudera.exe.prereq_jdk + # All variables will use their defaults, installing OpenJDK 17. + + - name: Set up Oracle JDK 11 without installing packages + ansible.builtin.import_role: + name: cloudera.exe.prereq_jdk + vars: + jdk_provider: oracle + jdk_version: 11 + jdk_install_packages: false # Assume JDK 11 is already installed + + - name: Set up OpenJDK 8 with JCE policy + ansible.builtin.import_role: + name: cloudera.exe.prereq_jdk + vars: + jdk_version: 8 + # Since version 8 is used, JCE enablement will be attempted. + jdk_security_paths: + - /etc/java/security/ + jdk_security_paths_override: false +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_jdk/defaults/main.yml b/roles/prereq_jdk/defaults/main.yml new file mode 100644 index 00000000..f76069e9 --- /dev/null +++ b/roles/prereq_jdk/defaults/main.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jdk_provider: openjdk +jdk_install_packages: true +# jdk_packages: [] +jdk_version: 17 +# jdk_security_paths: [] +jdk_security_paths_override: false diff --git a/roles/prereq_jdk/library/jdk_facts.py b/roles/prereq_jdk/library/jdk_facts.py new file mode 100644 index 00000000..051a2780 --- /dev/null +++ b/roles/prereq_jdk/library/jdk_facts.py @@ -0,0 +1,158 @@ +#!/usr/bin/python + +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = r""" +module: jdk_facts +short_description: Retrieve JDK information +description: + - Retrieve information about the installed Java JDK as facts. +options: {} +author: + - Webster Mudge +extends_documentation_fragment: + - action_common_attributes + - action_common_attributes.facts +attributes: + check_mode: + support: full + diff_mode: + support: none + facts: + support: full + platform: + support: full + seealso: + description: Java version history + link: https://en.wikipedia.org/wiki/Java_version_history +""" + +EXAMPLES = r""" +- name: Retrieve JDK details + cloudera.exe.jdk_facts: +""" + +RETURN = r""" +ansible_facts: + description: Facts to add to ansible_facts. + returned: always + type: complex + contains: + jdk: + description: + - Details on installed Java JDK executable. + returned: always + type: dict + contains: + provider: + description: + - JDK provider. + - Normalized for Oracle trademark. + - Returns C(None) if no JDK is discovered. + returned: always + version: + description: + - JDK version string in its entirety. + returned: when supported + major: + description: + - JDK C(major) version. + returned: when supported + minor: + description: + - JDK C(minor) version. + returned: when supported + patch: + description: + - JDK C(patch) version. + returned: when supported + release: + description: + - JDK C(release) version. + returned: when supported + build: + description: + - JDK C(build) version. + returned: when supported + update: + description: + - JDK C(update) details. + returned: when supported +""" + + +import re + +from ansible.module_utils.basic import AnsibleModule + +VERSION_REGEX = re.compile( + "(?P[\\w\\(\\)]+)" + + ".*" + + "\\(build\\s*" + + "(?P" + + "(?P\\d*)" + + "\\.?(?P\\d*)" + + "\\.?(?P\\d*)" + + "[+-_]?(?P[\\w\\d]*)" + + "[+-_]?(?P[\\w\\d]*)" + + ")" + + "\\)" +) + +UPDATE_REGEX = re.compile('".+"\\s*([\\w-]*)') + + +def main(): + result = dict( + ansible_facts=dict(jdk=dict()), + changed=False, + ) + + module = AnsibleModule(argument_spec=dict(), supports_check_mode=True) + + rc, _, stderr = module.run_command("/usr/bin/java -version") + if rc != 0: + module.warn(f"Unable to discover JDK facts: {stderr}") + result["ansible_facts"]["jdk"].update(provider=None) + else: + output = stderr.splitlines() + + version = VERSION_REGEX.search(output[1]) + update = UPDATE_REGEX.search(output[0]) + + result["ansible_facts"]["jdk"].update( + provider=( + "Oracle" + if version.group("provider") == "Java(TM)" + else version.group("provider") + ), + version=version.group("version"), + major=version.group("major"), + minor=version.group("minor"), + patch=version.group("patch"), + release=version.group("release"), + build=version.group("build"), + update=(update[1] if update is not None else ""), + ) + + module.exit_json(**result) + + +if __name__ == "__main__": + main() diff --git a/roles/prereq_jdk/meta/argument_specs.yml b/roles/prereq_jdk/meta/argument_specs.yml new file mode 100644 index 00000000..1d17fcee --- /dev/null +++ b/roles/prereq_jdk/meta/argument_specs.yml @@ -0,0 +1,61 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up JDK + description: + - Set up the Java Development Kit (JDK), optionally installing the JDK itself. + - For JDK 9 and below, optionally enable the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy. + - If the JDK is installed from the Cloudera repo, add any missing symlinks. + author: Cloudera Labs + options: + jdk_provider: + description: + - JDK vendor or provider. + default: openjdk + choices: + - openjdk + - oracle + - azul + jdk_install_packages: + description: + - Flag to enable JDK package installation. + type: bool + default: true + jdk_packages: + description: + - List of OS packages to install if I(jdk_install_packages=true). + type: list + elements: str + jdk_version: + description: + - Supported JDK version. + type: int + choices: + - 8 + - 11 + - 17 + default: 17 + jdk_security_paths: + description: + - List of paths to search for C(java.security) files to manage for JCE enablement. + type: list + elements: path + jdk_security_paths_override: + description: + - Flag to disable JCE changes ifo multiple C(java.security) files are found. + type: bool + default: false diff --git a/roles/prereq_jdk/molecule/default/converge.yml b/roles/prereq_jdk/molecule/default/converge.yml new file mode 100644 index 00000000..f745ce9e --- /dev/null +++ b/roles/prereq_jdk/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Set up JDK + ansible.builtin.import_role: + name: cloudera.exe.prereq_jdk diff --git a/roles/prereq_jdk/molecule/default/create.yml b/roles/prereq_jdk/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_jdk/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_jdk/molecule/default/destroy.yml b/roles/prereq_jdk/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_jdk/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_jdk/molecule/default/molecule.yml b/roles/prereq_jdk/molecule/default/molecule.yml new file mode 100644 index 00000000..c1dd32e3 --- /dev/null +++ b/roles/prereq_jdk/molecule/default/molecule.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_jdk-rhel9-4 + Project: Molecule testing for prereq_jdk +provisioner: + name: ansible diff --git a/roles/prereq_jdk/molecule/default/requirements.yml b/roles/prereq_jdk/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_jdk/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_jdk/molecule/default/verify.yml b/roles/prereq_jdk/molecule/default/verify.yml new file mode 100644 index 00000000..9c9219c8 --- /dev/null +++ b/roles/prereq_jdk/molecule/default/verify.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: diff --git a/roles/prereq_jdk/molecule/openjdk-8/converge.yml b/roles/prereq_jdk/molecule/openjdk-8/converge.yml new file mode 100644 index 00000000..75eee5ce --- /dev/null +++ b/roles/prereq_jdk/molecule/openjdk-8/converge.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Set up JDK + ansible.builtin.import_role: + name: cloudera.exe.prereq_jdk + vars: + jdk_version: 8 diff --git a/roles/prereq_jdk/molecule/openjdk-8/molecule.yml b/roles/prereq_jdk/molecule/openjdk-8/molecule.yml new file mode 100644 index 00000000..9c753ec4 --- /dev/null +++ b/roles/prereq_jdk/molecule/openjdk-8/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_jdk-8-rhel9-4 + Project: Molecule testing for prereq_jdk for JDK 8 +provisioner: + name: ansible + playbooks: + create: ../default/create.yml + destroy: ../default/destroy.yml diff --git a/roles/prereq_jdk/molecule/openjdk-8/requirements.yml b/roles/prereq_jdk/molecule/openjdk-8/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_jdk/molecule/openjdk-8/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_jdk/molecule/openjdk-8/verify.yml b/roles/prereq_jdk/molecule/openjdk-8/verify.yml new file mode 100644 index 00000000..9c9219c8 --- /dev/null +++ b/roles/prereq_jdk/molecule/openjdk-8/verify.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: diff --git a/roles/prereq_jdk/tasks/main.yml b/roles/prereq_jdk/tasks/main.yml new file mode 100644 index 00000000..f6f2c5a4 --- /dev/null +++ b/roles/prereq_jdk/tasks/main.yml @@ -0,0 +1,108 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install JDK + when: jdk_install_packages + ansible.builtin.include_tasks: "{{ jdk_provider }}_tasks.yml" + +- name: Check for Cloudera-repo JDK + ansible.builtin.stat: + path: /usr/java + register: __cldr_jdk + +- name: Set up JDK symlinks for Cloudera-repo JDK + when: __cldr_jdk.stat.exists + block: + - name: Discover the Cloudera Java binary + ansible.builtin.find: + paths: /usr/java + patterns: "jdk*-cloudera" + file_type: directory + recurse: false + register: __jdk_home + + - name: Create alternatives symlink for Cloudera Java binary + ansible.builtin.alternatives: + name: java + link: /usr/bin/java + path: "{{ __jdk_home.files[0].path }}/bin/java" + when: __jdk_home.matched + + - name: Create symlinks for Cloudera Java home directory + ansible.builtin.file: + src: "{{ __jdk_home.files[0].path }}" + dest: /usr/java/default + state: link + when: __jdk_home.matched + +- name: Discover installed JDK details + jdk_facts: + +- name: Enable JCE policy for JDK 9 or lower + when: ansible_facts.jdk["version"] is not version("10.0.0", ">=", version_type="loose") + block: + - name: Discover JDK java.security files + ansible.builtin.find: + paths: "{{ jdk_security_paths | default(__java_security_paths[jdk_version]) }}" + pattern: "java.security" + follow: true + recurse: true + register: __java_security + + - name: Check JDK java.security files + ansible.builtin.fail: + msg: > + Multiple copies of java.security were found. Exiting to avoid editing the incorrect one. + To override, set variable "jdk_security_paths_override=True". + when: __java_security.matched > 1 and not jdk_security_paths_override + + - name: Enable Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy + ansible.builtin.lineinfile: + path: "{{ jce_file.path }}" + regexp: "#?crypto.policy=" + line: crypto.policy=unlimited + loop: "{{ __java_security.files }}" + loop_control: + loop_var: jce_file + +- name: Patch Kerberos cross-realm referrals (JDK-8215032) + ansible.builtin.lineinfile: + path: "{{ krb_file.path }}" + regexp: "^sun.security.krb5.disableReferrals=" + line: sun.security.krb5.disableReferrals=true + loop: "{{ __java_security.files }}" + loop_control: + loop_var: krb_file + when: > + ansible_facts.jdk["provider"] == "openjdk" and + ( + (ansible_facts.jdk["version"] is version("1.8", ">=", version_type="loose") and ansible_facts.jdk["version"] is version("1.9", "<=", version_type="loose") and + ansible_facts.jdk["update"]|int >= 242) + or + (ansible_facts.jdk["version"] is version("11.0.6", ">=", version_type="semver")) + ) + +# https://docs.cloudera.com/cdp-private-cloud-base/7.1.9/installation/topics/cdpdc-manually-installing-openjdk.html diff --git a/roles/prereq_jdk/tasks/openjdk_tasks.yml b/roles/prereq_jdk/tasks/openjdk_tasks.yml new file mode 100644 index 00000000..46f52ae5 --- /dev/null +++ b/roles/prereq_jdk/tasks/openjdk_tasks.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install OpenJDK + ansible.builtin.package: + lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}" + name: "{{ jdk_package | default(__openjdk_package[jdk_version]) }}" + state: present diff --git a/roles/prereq_jdk/vars/RedHat.yml b/roles/prereq_jdk/vars/RedHat.yml new file mode 100644 index 00000000..acc1934d --- /dev/null +++ b/roles/prereq_jdk/vars/RedHat.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__openjdk_package: + 8: java-1.8.0-openjdk-devel + 11: java-11-openjdk-devel + 17: java-17-openjdk-devel + +__java_security_paths: + 8: + - /etc/java + 11: + - /etc/java + 17: + - /etc/java diff --git a/roles/prereq_jdk/vars/default.yml b/roles/prereq_jdk/vars/default.yml new file mode 100644 index 00000000..351eb094 --- /dev/null +++ b/roles/prereq_jdk/vars/default.yml @@ -0,0 +1,13 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/roles/prereq_jdk/vars/main.yml b/roles/prereq_jdk/vars/main.yml new file mode 100644 index 00000000..351eb094 --- /dev/null +++ b/roles/prereq_jdk/vars/main.yml @@ -0,0 +1,13 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From a8c07df91c4a9d6093bb7daac397cd53a3cf1260 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:58 -0400 Subject: [PATCH 24/72] Add Apache Kafka prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_kafka/README.md | 53 +++ roles/prereq_kafka/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_kafka/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_kafka/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_kafka/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_kafka/molecule/default/verify.yml | 36 ++ roles/prereq_kafka/tasks/main.yml | 39 ++ roles/prereq_kafka/vars/main.yml | 24 ++ 11 files changed, 798 insertions(+) create mode 100644 roles/prereq_kafka/README.md create mode 100644 roles/prereq_kafka/meta/argument_specs.yml create mode 100644 roles/prereq_kafka/molecule/default/converge.yml create mode 100644 roles/prereq_kafka/molecule/default/create.yml create mode 100644 roles/prereq_kafka/molecule/default/destroy.yml create mode 100644 roles/prereq_kafka/molecule/default/molecule.yml create mode 100644 roles/prereq_kafka/molecule/default/prepare.yml create mode 100644 roles/prereq_kafka/molecule/default/requirements.yml create mode 100644 roles/prereq_kafka/molecule/default/verify.yml create mode 100644 roles/prereq_kafka/tasks/main.yml create mode 100644 roles/prereq_kafka/vars/main.yml diff --git a/roles/prereq_kafka/README.md b/roles/prereq_kafka/README.md new file mode 100644 index 00000000..465c37c8 --- /dev/null +++ b/roles/prereq_kafka/README.md @@ -0,0 +1,53 @@ +# prereq_kafka + +Set up for Kafka + +This role prepares a host for Apache Kafka usage by creating dedicated system users and groups for both `kafka` and `cruisecontrol`. These users are essential for running their respective services with appropriate permissions and isolation. + +The role will: +- Create the `kafka` system user and group. +- Create the `cruisecontrol` system user and group. +- Configure home directories and other necessary local paths for these users, if required. +- Ensure appropriate permissions are set for files and directories related to Kafka and Cruise Control. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: kafka_nodes + tasks: + - name: Set up the kafka and cruisecontrol users and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_kafka +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_kafka/meta/argument_specs.yml b/roles/prereq_kafka/meta/argument_specs.yml new file mode 100644 index 00000000..8bf6c914 --- /dev/null +++ b/roles/prereq_kafka/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Kafka + description: | + Set up for Kafka usage, notably, create the local C(kafka) and C(cruisecontrol) users. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_kafka/molecule/default/converge.yml b/roles/prereq_kafka/molecule/default/converge.yml new file mode 100644 index 00000000..b7dba075 --- /dev/null +++ b/roles/prereq_kafka/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Kafka + ansible.builtin.import_role: + name: cloudera.exe.prereq_kafka diff --git a/roles/prereq_kafka/molecule/default/create.yml b/roles/prereq_kafka/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_kafka/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_kafka/molecule/default/destroy.yml b/roles/prereq_kafka/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_kafka/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_kafka/molecule/default/molecule.yml b/roles/prereq_kafka/molecule/default/molecule.yml new file mode 100644 index 00000000..558d990c --- /dev/null +++ b/roles/prereq_kafka/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kafka-rhel9-4 + Project: Molecule testing for prereq_kafka +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_kafka/molecule/default/prepare.yml b/roles/prereq_kafka/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_kafka/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_kafka/molecule/default/requirements.yml b/roles/prereq_kafka/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_kafka/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_kafka/molecule/default/verify.yml b/roles/prereq_kafka/molecule/default/verify.yml new file mode 100644 index 00000000..d67e46b5 --- /dev/null +++ b/roles/prereq_kafka/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for kafka users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ kafka_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ kafka_local_accounts }}" diff --git a/roles/prereq_kafka/tasks/main.yml b/roles/prereq_kafka/tasks/main.yml new file mode 100644 index 00000000..918adc84 --- /dev/null +++ b/roles/prereq_kafka/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ kafka_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ kafka_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_kafka/vars/main.yml b/roles/prereq_kafka/vars/main.yml new file mode 100644 index 00000000..d51922e7 --- /dev/null +++ b/roles/prereq_kafka/vars/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kafka_local_accounts: + - user: cruisecontrol + home: /var/lib/cruise_control + comment: Cruise Control + keystore_acl: true + - user: kafka + home: /var/lib/kafka + comment: Kafka + keystore_acl: true From 56e3d57e6319cd5c14ad9737a69b145c7fcb87d8 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:42:59 -0400 Subject: [PATCH 25/72] Add Kerberos prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_kerberos/README.md | 73 ++++ roles/prereq_kerberos/defaults/main.yml | 22 ++ roles/prereq_kerberos/handlers/main.yml | 19 + roles/prereq_kerberos/meta/argument_specs.yml | 53 +++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 43 +++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_kerberos/tasks/main.yml | 82 +++++ roles/prereq_kerberos/vars/RedHat.yml | 18 + roles/prereq_kerberos/vars/Ubuntu.yml | 17 + roles/prereq_kerberos/vars/default.yml | 13 + 13 files changed, 877 insertions(+) create mode 100644 roles/prereq_kerberos/README.md create mode 100644 roles/prereq_kerberos/defaults/main.yml create mode 100644 roles/prereq_kerberos/handlers/main.yml create mode 100644 roles/prereq_kerberos/meta/argument_specs.yml create mode 100644 roles/prereq_kerberos/molecule/default/converge.yml create mode 100644 roles/prereq_kerberos/molecule/default/create.yml create mode 100644 roles/prereq_kerberos/molecule/default/destroy.yml create mode 100644 roles/prereq_kerberos/molecule/default/molecule.yml create mode 100644 roles/prereq_kerberos/molecule/default/requirements.yml create mode 100644 roles/prereq_kerberos/tasks/main.yml create mode 100644 roles/prereq_kerberos/vars/RedHat.yml create mode 100644 roles/prereq_kerberos/vars/Ubuntu.yml create mode 100644 roles/prereq_kerberos/vars/default.yml diff --git a/roles/prereq_kerberos/README.md b/roles/prereq_kerberos/README.md new file mode 100644 index 00000000..f27baa41 --- /dev/null +++ b/roles/prereq_kerberos/README.md @@ -0,0 +1,73 @@ +# prereq_kerberos + +Set up for Kerberos + +This role prepares a host for Kerberos usage by installing the necessary OS-specific client libraries. It configures the Kerberos credential cache to use KCM (Kerberos Credential Manager) and can optionally set up and configure the SSSD (System Security Services Daemon) for user authentication and authorization. + +The role will: +- Install a list of specified Kerberos client packages. If not provided, it will determine and install the appropriate packages based on the target host's operating system. +- Configure the Kerberos client by managing the `krb5.conf` file, including setting the `kerberos_realm`. +- Configure the Kerberos Credential Manager (KCM) to be the default credential cache. +- If SSSD is used, the role will configure the `sssd.conf` file and manage the `sssd` service. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage packages and system configuration files. +- Network access to package repositories. +- A functional Kerberos Key Distribution Center (KDC) is assumed to be available on the network to service the specified `kerberos_realm`. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `kerberos_packages` | `list` of `str` | `False` | `[defaults based on OS]` | List of Kerberos client packages to install. If not defined, the role will install default packages based on the OS distribution. | +| `kerberos_config_path` | `path` | `False` | `/etc/krb5.conf` | Path to the main Kerberos configuration file. | +| `kerberos_kcm_credential_cache_config_path` | `path` | `False` | `/etc/krb5.conf.d/kcm_default_ccache` | Path to the configuration file that sets the default credential cache type to KCM. | +| `kerberos_realm` | `str` | `True` | | The name of the Kerberos realm to which the host will belong. This is a mandatory parameter. | +| `sssd_config_path` | `path` | `False` | `/etc/sssd/sssd.conf` | Path to the SSSD configuration file. The role will only manage this file if SSSD is part of the overall setup. | +| `sssd_service` | `str` | `False` | `sssd` | The name of the SSSD service to manage. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Set up Kerberos client for the example.internal realm + ansible.builtin.import_role: + name: cloudera.exe.prereq_kerberos + vars: + kerberos_realm: "EXAMPLE.INTERNAL" + # All other options will use their defaults, including packages and SSSD settings. + + - name: Set up Kerberos with custom paths and SSSD configuration + ansible.builtin.import_role: + name: cloudera.exe.prereq_kerberos + vars: + kerberos_realm: "MY.CUSTOM.REALM" + kerberos_packages: + - krb5-workstation # Example custom package + kerberos_config_path: "/etc/kerberos/krb5.conf" + sssd_config_path: "/etc/sssd/my_sssd.conf" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_kerberos/defaults/main.yml b/roles/prereq_kerberos/defaults/main.yml new file mode 100644 index 00000000..fd199d40 --- /dev/null +++ b/roles/prereq_kerberos/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# kerberos_packages: [] +kerberos_config_path: "/etc/krb5.conf" +kerberos_kcm_credential_cache_config_path: "/etc/krb5.conf.d/kcm_default_ccache" +kerberos_realm: "{{ undef(hint='Please provide the Kerberos realm') }}" + +sssd_config_path: "/etc/sssd/sssd.conf" +sssd_service: sssd diff --git a/roles/prereq_kerberos/handlers/main.yml b/roles/prereq_kerberos/handlers/main.yml new file mode 100644 index 00000000..fa5d7802 --- /dev/null +++ b/roles/prereq_kerberos/handlers/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Restart SSSD + ansible.builtin.service: + name: "{{ sssd_service }}" + state: restarted diff --git a/roles/prereq_kerberos/meta/argument_specs.yml b/roles/prereq_kerberos/meta/argument_specs.yml new file mode 100644 index 00000000..7c87e94e --- /dev/null +++ b/roles/prereq_kerberos/meta/argument_specs.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Kerberos + description: + - Set up for Kerberos usage, including OS-specific Kerberos client libraries. + - Configure Kerberos credential cache. + - Configure SSSD, if needed. + author: Cloudera Labs + options: + kerberos_packages: + description: + - List of Kerberos client packages to install. + - If not defined, will default to OS-specific packages. + type: list + elements: str + kerberos_config_path: + description: + - Path to the Kerberos configuration file. + type: path + default: "/etc/krb5.conf" + kerberos_kcm_credential_cache_config_path: + description: + - Path to the KCM configuration file. + type: path + default: "/etc/krb5.conf.d/kcm_default_ccache" + kerberos_realm: + description: + - Name of the Kerberos realm. + required: true + sssd_config_path: + description: + - Path to the SSSD configuration file. + type: path + default: "/etc/sssd/sssd.conf" + sssd_service: + description: + - Name of the SSSD service. + default: sssd diff --git a/roles/prereq_kerberos/molecule/default/converge.yml b/roles/prereq_kerberos/molecule/default/converge.yml new file mode 100644 index 00000000..cf406e48 --- /dev/null +++ b/roles/prereq_kerberos/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Configure for Kerberos + ansible.builtin.import_role: + name: cloudera.exe.prereq_kerberos diff --git a/roles/prereq_kerberos/molecule/default/create.yml b/roles/prereq_kerberos/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_kerberos/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_kerberos/molecule/default/destroy.yml b/roles/prereq_kerberos/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_kerberos/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_kerberos/molecule/default/molecule.yml b/roles/prereq_kerberos/molecule/default/molecule.yml new file mode 100644 index 00000000..73c69fca --- /dev/null +++ b/roles/prereq_kerberos/molecule/default/molecule.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kerberos-rhel9-4 + Project: Molecule testing for prereq_kerberos +provisioner: + name: ansible + inventory: + group_vars: + all: + kerberos_realm: EXAMPLE.INTERNAL diff --git a/roles/prereq_kerberos/molecule/default/requirements.yml b/roles/prereq_kerberos/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_kerberos/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_kerberos/tasks/main.yml b/roles/prereq_kerberos/tasks/main.yml new file mode 100644 index 00000000..a41bc897 --- /dev/null +++ b/roles/prereq_kerberos/tasks/main.yml @@ -0,0 +1,82 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install Kerberos packages + ansible.builtin.package: + name: "{{ kerberos_packages | default(__kerberos_packages) }}" + state: present + +- name: Update renewable Kerberos ticket lifetime + community.general.ini_file: + path: "{{ kerberos_config_path }}" + section: libdefaults + option: "{{ __conf.key }}" + value: "{{ __conf.value }}" + mode: "0755" + loop: "{{ updates | dict2items }}" + loop_control: + loop_var: __conf + label: "{{ __conf.key }}" + vars: + updates: + renew_lifetime: 7d + max_life: 365d + max_renewable_life: 365d + +- name: Remove default Kerberos credential cache option + community.general.ini_file: + path: "{{ krb_ccache }}" + section: libdefaults + option: default_ccache_name + state: absent + mode: "0755" + loop: + - "{{ kerberos_config_path }}" + - "{{ kerberos_kcm_credential_cache_config_path }}" + loop_control: + loop_var: krb_ccache + +- name: Get stats for SSSD configuration + ansible.builtin.stat: + path: "{{ sssd_config_path }}" + register: __sssd + +- name: Update SSSD to cache Kerberos tickets as files + when: __sssd.stat.exists + community.general.ini_file: + path: "{{ sssd_config_path }}" + section: "domain/{{ kerberos_realm | lower }}" + option: "{{ __sssd_conf.key }}" + value: "{{ __sssd_conf.value | string }}" + mode: "0755" + loop: "{{ entries | dict2items }}" + loop_control: + loop_var: __sssd_conf + label: "{{ __sssd_conf.key }}" + vars: + entries: + krb5_ccname_template: "FILE:/tmp/krb5cc_%U_XXXXXX" + notify: Restart SSSD diff --git a/roles/prereq_kerberos/vars/RedHat.yml b/roles/prereq_kerberos/vars/RedHat.yml new file mode 100644 index 00000000..87f50b9a --- /dev/null +++ b/roles/prereq_kerberos/vars/RedHat.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__kerberos_packages: + - krb5-workstation + - krb5-libs diff --git a/roles/prereq_kerberos/vars/Ubuntu.yml b/roles/prereq_kerberos/vars/Ubuntu.yml new file mode 100644 index 00000000..d535242f --- /dev/null +++ b/roles/prereq_kerberos/vars/Ubuntu.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__kerberos_packages: + - krb5-user diff --git a/roles/prereq_kerberos/vars/default.yml b/roles/prereq_kerberos/vars/default.yml new file mode 100644 index 00000000..351eb094 --- /dev/null +++ b/roles/prereq_kerberos/vars/default.yml @@ -0,0 +1,13 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From ccb2defde424f9b8bca950542bd0945a8c682bc1 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:00 -0400 Subject: [PATCH 26/72] Add kernel prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_kernel/README.md | 61 ++++ roles/prereq_kernel/defaults/main.yml | 23 ++ roles/prereq_kernel/handlers/main.yml | 16 + roles/prereq_kernel/meta/argument_specs.yml | 34 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_kernel/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 51 +++ .../molecule/default/requirements.yml | 21 ++ .../prereq_kernel/molecule/default/verify.yml | 34 ++ roles/prereq_kernel/tasks/main.yml | 23 ++ roles/prereq_kernel/tests/inventory | 15 + roles/prereq_kernel/tests/test.yml | 20 ++ roles/prereq_kernel/vars/main.yml | 16 + 14 files changed, 830 insertions(+) create mode 100644 roles/prereq_kernel/README.md create mode 100644 roles/prereq_kernel/defaults/main.yml create mode 100644 roles/prereq_kernel/handlers/main.yml create mode 100644 roles/prereq_kernel/meta/argument_specs.yml create mode 100644 roles/prereq_kernel/molecule/default/converge.yml create mode 100644 roles/prereq_kernel/molecule/default/create.yml create mode 100644 roles/prereq_kernel/molecule/default/destroy.yml create mode 100644 roles/prereq_kernel/molecule/default/molecule.yml create mode 100644 roles/prereq_kernel/molecule/default/requirements.yml create mode 100644 roles/prereq_kernel/molecule/default/verify.yml create mode 100644 roles/prereq_kernel/tasks/main.yml create mode 100644 roles/prereq_kernel/tests/inventory create mode 100644 roles/prereq_kernel/tests/test.yml create mode 100644 roles/prereq_kernel/vars/main.yml diff --git a/roles/prereq_kernel/README.md b/roles/prereq_kernel/README.md new file mode 100644 index 00000000..a4fd15cc --- /dev/null +++ b/roles/prereq_kernel/README.md @@ -0,0 +1,61 @@ +# prereq_kernel + +Update kernel parameters + +This role updates and applies specified kernel parameters using `sysctl`. It is designed to configure a host for performance-critical applications by tuning memory management settings like `swappiness` and `overcommit_memory`, and by disabling IPv6 functionality for all network interfaces by default. The changes are applied both for the current running system and for future reboots. + +The role will: +- Modify kernel parameters via `sysctl`. +- Ensure the changes are made persistent by writing them to a configuration file (e.g., in `/etc/sysctl.d/`). +- By default, it will configure memory management settings and disable IPv6. + +# Requirements + +- Root or `sudo` privileges are required on the target host to update kernel parameters. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `prereq_kernel__kernel_flags` | `dict` | `False` | `{"vm.swappiness": 1, "vm.overcommit_memory": 1, "net.ipv6.conf.all.disable_ipv6": 1, "net.ipv6.conf.default.disable_ipv6": 1, "net.ipv6.conf.lo.disable_ipv6": 1}` | A dictionary of kernel parameters and their values to configure with `sysctl`. The role will apply the default values unless overridden. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Update kernel parameters with default values + ansible.builtin.import_role: + name: cloudera.exe.prereq_kernel + # This will set vm.swappiness, overcommit_memory, and disable IPv6 as per the default values. + + - name: Override default swappiness and enable IPv6 on the loopback interface + ansible.builtin.import_role: + name: cloudera.exe.prereq_kernel + vars: + prereq_kernel__kernel_flags: + vm.swappiness: 10 + net.ipv6.conf.lo.disable_ipv6: 0 # Enable IPv6 on loopback +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_kernel/defaults/main.yml b/roles/prereq_kernel/defaults/main.yml new file mode 100644 index 00000000..979eb387 --- /dev/null +++ b/roles/prereq_kernel/defaults/main.yml @@ -0,0 +1,23 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# defaults file for prereq_kernel + +kernel_flags: + vm.swappiness: 1 + vm.overcommit_memory: 1 + net.ipv6.conf.all.disable_ipv6: 1 + net.ipv6.conf.default.disable_ipv6: 1 + net.ipv6.conf.lo.disable_ipv6: 1 diff --git a/roles/prereq_kernel/handlers/main.yml b/roles/prereq_kernel/handlers/main.yml new file mode 100644 index 00000000..99029163 --- /dev/null +++ b/roles/prereq_kernel/handlers/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# handlers file for prereq_kernel diff --git a/roles/prereq_kernel/meta/argument_specs.yml b/roles/prereq_kernel/meta/argument_specs.yml new file mode 100644 index 00000000..88c99ed6 --- /dev/null +++ b/roles/prereq_kernel/meta/argument_specs.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Update kernel parameters + description: | + Updates and applies kernel parameters using C(sysctl). + This includes controlling memory management (swappiness and memory overcommit) + and configuring IPv6 settings to disable IPv6 functionality. + author: + - "Ronald Suplina " + options: + prereq_kernel__kernel_flags: + description: Dictionary of kernel parameters to configure with C(sysctl). + type: "dict" + default: + vm.swappiness: 1 + vm.overcommit_memory: 1 + net.ipv6.conf.all.disable_ipv6: 1 + net.ipv6.conf.default.disable_ipv6: 1 + net.ipv6.conf.lo.disable_ipv6: 1 diff --git a/roles/prereq_kernel/molecule/default/converge.yml b/roles/prereq_kernel/molecule/default/converge.yml new file mode 100644 index 00000000..9dc4e0ef --- /dev/null +++ b/roles/prereq_kernel/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Update Kernel permissions + ansible.builtin.import_role: + name: cloudera.exe.prereq_kernel diff --git a/roles/prereq_kernel/molecule/default/create.yml b/roles/prereq_kernel/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_kernel/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_kernel/molecule/default/destroy.yml b/roles/prereq_kernel/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_kernel/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_kernel/molecule/default/molecule.yml b/roles/prereq_kernel/molecule/default/molecule.yml new file mode 100644 index 00000000..33f82ee4 --- /dev/null +++ b/roles/prereq_kernel/molecule/default/molecule.yml @@ -0,0 +1,51 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kernel-rhel9-4 + Project: Molecule testing for prereq_kernel + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kernel-ubuntu20-04 + Project: Molecule testing for prereq_kernel +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_kernel/molecule/default/requirements.yml b/roles/prereq_kernel/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_kernel/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_kernel/molecule/default/verify.yml b/roles/prereq_kernel/molecule/default/verify.yml new file mode 100644 index 00000000..08cb24a1 --- /dev/null +++ b/roles/prereq_kernel/molecule/default/verify.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Set kernel flags for verification + ansible.builtin.set_fact: + kernel_flags: + vm.swappiness: "1" + vm.overcommit_memory: "1" + net.ipv6.conf.all.disable_ipv6: "1" + net.ipv6.conf.default.disable_ipv6: "1" + net.ipv6.conf.lo.disable_ipv6: "1" + + - name: Verify kernel parameters are set to 1 + ansible.builtin.command: "sysctl -n {{ item.key }}" + register: __kernel_flag + failed_when: __kernel_flag.stdout.strip() != item.value + changed_when: false + with_dict: "{{ kernel_flags }}" diff --git a/roles/prereq_kernel/tasks/main.yml b/roles/prereq_kernel/tasks/main.yml new file mode 100644 index 00000000..8fdb7b0b --- /dev/null +++ b/roles/prereq_kernel/tasks/main.yml @@ -0,0 +1,23 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Update kernel parameters + ansible.posix.sysctl: + name: "{{ item.key }}" + value: "{{ item.value }}" + state: present + sysctl_set: true + reload: true + with_dict: "{{ kernel_flags }}" diff --git a/roles/prereq_kernel/tests/inventory b/roles/prereq_kernel/tests/inventory new file mode 100644 index 00000000..6778d06c --- /dev/null +++ b/roles/prereq_kernel/tests/inventory @@ -0,0 +1,15 @@ +// Copyright 2024 Cloudera, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +localhost diff --git a/roles/prereq_kernel/tests/test.yml b/roles/prereq_kernel/tests/test.yml new file mode 100644 index 00000000..085585d8 --- /dev/null +++ b/roles/prereq_kernel/tests/test.yml @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Test + hosts: localhost + remote_user: root + roles: + - prereq_kernel diff --git a/roles/prereq_kernel/vars/main.yml b/roles/prereq_kernel/vars/main.yml new file mode 100644 index 00000000..51f834ff --- /dev/null +++ b/roles/prereq_kernel/vars/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# vars file for prereq_kernel From 946b1c69c964f1df7e8e3499e4f2f2c13c9fd62b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:00 -0400 Subject: [PATCH 27/72] Add Cloudera Navigator Key Trustee prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_keytrustee/README.md | 52 +++ .../prereq_keytrustee/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_keytrustee/tasks/main.yml | 39 ++ roles/prereq_keytrustee/vars/main.yml | 22 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_keytrustee/README.md create mode 100644 roles/prereq_keytrustee/meta/argument_specs.yml create mode 100644 roles/prereq_keytrustee/molecule/default/converge.yml create mode 100644 roles/prereq_keytrustee/molecule/default/create.yml create mode 100644 roles/prereq_keytrustee/molecule/default/destroy.yml create mode 100644 roles/prereq_keytrustee/molecule/default/molecule.yml create mode 100644 roles/prereq_keytrustee/molecule/default/prepare.yml create mode 100644 roles/prereq_keytrustee/molecule/default/requirements.yml create mode 100644 roles/prereq_keytrustee/molecule/default/verify.yml create mode 100644 roles/prereq_keytrustee/tasks/main.yml create mode 100644 roles/prereq_keytrustee/vars/main.yml diff --git a/roles/prereq_keytrustee/README.md b/roles/prereq_keytrustee/README.md new file mode 100644 index 00000000..ae2c192c --- /dev/null +++ b/roles/prereq_keytrustee/README.md @@ -0,0 +1,52 @@ +# prereq_keytrustee + +Set up for Key Trustee Server + +This role prepares a host for Cloudera Navigator Key Trustee server usage by creating a dedicated system user and group named `keytrustee`. This user is essential for running Key Trustee processes with appropriate permissions and isolation. + +The role will: +- Create the `keytrustee` system user and group. +- Configure home directories and other necessary local paths for the `keytrustee` user, if required. +- Ensure appropriate permissions are set for files and directories related to Key Trustee. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: keytrustee_nodes + tasks: + - name: Set up the keytrustee user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_keytrustee +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_keytrustee/meta/argument_specs.yml b/roles/prereq_keytrustee/meta/argument_specs.yml new file mode 100644 index 00000000..8735250f --- /dev/null +++ b/roles/prereq_keytrustee/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Key Trustee + description: | + Set up for Cloudera Navigator Key Trustee usage, notably, create the local C(keytrustee) users. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_keytrustee/molecule/default/converge.yml b/roles/prereq_keytrustee/molecule/default/converge.yml new file mode 100644 index 00000000..ae02e6b4 --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Key Trustee Server + ansible.builtin.import_role: + name: cloudera.exe.prereq_keytrustee diff --git a/roles/prereq_keytrustee/molecule/default/create.yml b/roles/prereq_keytrustee/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_keytrustee/molecule/default/destroy.yml b/roles/prereq_keytrustee/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_keytrustee/molecule/default/molecule.yml b/roles/prereq_keytrustee/molecule/default/molecule.yml new file mode 100644 index 00000000..856e5a6c --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_keytrustee-rhel9-4 + Project: Molecule testing for prereq_keytrustee +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_keytrustee/molecule/default/prepare.yml b/roles/prereq_keytrustee/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_keytrustee/molecule/default/requirements.yml b/roles/prereq_keytrustee/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_keytrustee/molecule/default/verify.yml b/roles/prereq_keytrustee/molecule/default/verify.yml new file mode 100644 index 00000000..b1c0ae8c --- /dev/null +++ b/roles/prereq_keytrustee/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for Key Trustee Server users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ keytrustee_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ keytrustee_local_accounts }}" diff --git a/roles/prereq_keytrustee/tasks/main.yml b/roles/prereq_keytrustee/tasks/main.yml new file mode 100644 index 00000000..0cb088ea --- /dev/null +++ b/roles/prereq_keytrustee/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ keytrustee_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ keytrustee_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_keytrustee/vars/main.yml b/roles/prereq_keytrustee/vars/main.yml new file mode 100644 index 00000000..89462ebe --- /dev/null +++ b/roles/prereq_keytrustee/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +keytrustee_local_accounts: + - user: keytrustee + home: /var/lib/keytrustee + comment: KeyTrustee KMS + keystore_acl: true + key_acl: true + key_password_acl: true From e264a23dc9e0714a22ae41cfe7d28409e967302c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:01 -0400 Subject: [PATCH 28/72] Add Cloudera Key Management Systems prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_kms/README.md | 52 +++ roles/prereq_kms/meta/argument_specs.yml | 22 ++ .../prereq_kms/molecule/default/converge.yml | 23 ++ roles/prereq_kms/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_kms/molecule/default/destroy.yml | 157 ++++++++ .../prereq_kms/molecule/default/molecule.yml | 49 +++ roles/prereq_kms/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_kms/molecule/default/verify.yml | 36 ++ roles/prereq_kms/tasks/main.yml | 39 ++ roles/prereq_kms/vars/main.yml | 20 ++ 11 files changed, 793 insertions(+) create mode 100644 roles/prereq_kms/README.md create mode 100644 roles/prereq_kms/meta/argument_specs.yml create mode 100644 roles/prereq_kms/molecule/default/converge.yml create mode 100644 roles/prereq_kms/molecule/default/create.yml create mode 100644 roles/prereq_kms/molecule/default/destroy.yml create mode 100644 roles/prereq_kms/molecule/default/molecule.yml create mode 100644 roles/prereq_kms/molecule/default/prepare.yml create mode 100644 roles/prereq_kms/molecule/default/requirements.yml create mode 100644 roles/prereq_kms/molecule/default/verify.yml create mode 100644 roles/prereq_kms/tasks/main.yml create mode 100644 roles/prereq_kms/vars/main.yml diff --git a/roles/prereq_kms/README.md b/roles/prereq_kms/README.md new file mode 100644 index 00000000..db2c2001 --- /dev/null +++ b/roles/prereq_kms/README.md @@ -0,0 +1,52 @@ +# prereq_kms + +Set up for KMS + +This role prepares a host for Cloudera Key Management System (KMS) usage by creating a dedicated system user and group named `kms`. This user is essential for running KMS processes with appropriate permissions and isolation. + +The role will: +- Create the `kms` system user and group. +- Configure home directories and other necessary local paths for the `kms` user, if required. +- Ensure appropriate permissions are set for files and directories related to KMS. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: kms_nodes + tasks: + - name: Set up the kms user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_kms +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_kms/meta/argument_specs.yml b/roles/prereq_kms/meta/argument_specs.yml new file mode 100644 index 00000000..6621294a --- /dev/null +++ b/roles/prereq_kms/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for KMS + description: | + Set up for Cloudera Key Management System (KMS) usage, notably, create the local C(kms) users. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_kms/molecule/default/converge.yml b/roles/prereq_kms/molecule/default/converge.yml new file mode 100644 index 00000000..80700fba --- /dev/null +++ b/roles/prereq_kms/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for KMS + ansible.builtin.import_role: + name: cloudera.exe.prereq_kms diff --git a/roles/prereq_kms/molecule/default/create.yml b/roles/prereq_kms/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_kms/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_kms/molecule/default/destroy.yml b/roles/prereq_kms/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_kms/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_kms/molecule/default/molecule.yml b/roles/prereq_kms/molecule/default/molecule.yml new file mode 100644 index 00000000..448199f2 --- /dev/null +++ b/roles/prereq_kms/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kms-rhel9-4 + Project: Molecule testing for prereq_kms +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_kms/molecule/default/prepare.yml b/roles/prereq_kms/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_kms/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_kms/molecule/default/requirements.yml b/roles/prereq_kms/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_kms/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_kms/molecule/default/verify.yml b/roles/prereq_kms/molecule/default/verify.yml new file mode 100644 index 00000000..50a36aa6 --- /dev/null +++ b/roles/prereq_kms/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for KMS users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ kms_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ kms_local_accounts }}" diff --git a/roles/prereq_kms/tasks/main.yml b/roles/prereq_kms/tasks/main.yml new file mode 100644 index 00000000..ada79d17 --- /dev/null +++ b/roles/prereq_kms/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ kms_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ kms_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_kms/vars/main.yml b/roles/prereq_kms/vars/main.yml new file mode 100644 index 00000000..25f8974d --- /dev/null +++ b/roles/prereq_kms/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kms_local_accounts: + - user: kms + home: /var/lib/hadoop-kms + comment: Hadoop KMS + keystore_acl: true From 96ebc7c14fad1317682c5e6206e124452e887a70 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:02 -0400 Subject: [PATCH 29/72] Add Apache Knox prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_knox/README.md | 52 +++ roles/prereq_knox/meta/argument_specs.yml | 22 ++ .../prereq_knox/molecule/default/converge.yml | 23 ++ roles/prereq_knox/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_knox/molecule/default/destroy.yml | 157 ++++++++ .../prereq_knox/molecule/default/molecule.yml | 49 +++ .../prereq_knox/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_knox/molecule/default/verify.yml | 36 ++ roles/prereq_knox/tasks/main.yml | 39 ++ roles/prereq_knox/vars/main.yml | 21 ++ 11 files changed, 794 insertions(+) create mode 100644 roles/prereq_knox/README.md create mode 100644 roles/prereq_knox/meta/argument_specs.yml create mode 100644 roles/prereq_knox/molecule/default/converge.yml create mode 100644 roles/prereq_knox/molecule/default/create.yml create mode 100644 roles/prereq_knox/molecule/default/destroy.yml create mode 100644 roles/prereq_knox/molecule/default/molecule.yml create mode 100644 roles/prereq_knox/molecule/default/prepare.yml create mode 100644 roles/prereq_knox/molecule/default/requirements.yml create mode 100644 roles/prereq_knox/molecule/default/verify.yml create mode 100644 roles/prereq_knox/tasks/main.yml create mode 100644 roles/prereq_knox/vars/main.yml diff --git a/roles/prereq_knox/README.md b/roles/prereq_knox/README.md new file mode 100644 index 00000000..cc5b3144 --- /dev/null +++ b/roles/prereq_knox/README.md @@ -0,0 +1,52 @@ +# prereq_knox + +Set up for Knox + +This role prepares a host for Apache Knox usage by creating a dedicated system user and group named `knox`. This user is essential for running Knox processes with appropriate permissions and isolation. + +The role will: +- Create the `knox` system user and group. +- Configure home directories and other necessary local paths for the `knox` user, if required. +- Ensure appropriate permissions are set for files and directories related to Knox. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: knox_nodes + tasks: + - name: Set up the knox user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_knox +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_knox/meta/argument_specs.yml b/roles/prereq_knox/meta/argument_specs.yml new file mode 100644 index 00000000..f816cda1 --- /dev/null +++ b/roles/prereq_knox/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Knox + description: | + Set up for Apache Knox usage, notably, create the local C(knox) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_knox/molecule/default/converge.yml b/roles/prereq_knox/molecule/default/converge.yml new file mode 100644 index 00000000..b49ed0ee --- /dev/null +++ b/roles/prereq_knox/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Knox + ansible.builtin.import_role: + name: cloudera.exe.prereq_knox diff --git a/roles/prereq_knox/molecule/default/create.yml b/roles/prereq_knox/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_knox/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_knox/molecule/default/destroy.yml b/roles/prereq_knox/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_knox/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_knox/molecule/default/molecule.yml b/roles/prereq_knox/molecule/default/molecule.yml new file mode 100644 index 00000000..48762f43 --- /dev/null +++ b/roles/prereq_knox/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_knox-rhel9-4 + Project: Molecule testing for prereq_knox +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_knox/molecule/default/prepare.yml b/roles/prereq_knox/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_knox/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_knox/molecule/default/requirements.yml b/roles/prereq_knox/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_knox/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_knox/molecule/default/verify.yml b/roles/prereq_knox/molecule/default/verify.yml new file mode 100644 index 00000000..4ca6ae8e --- /dev/null +++ b/roles/prereq_knox/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for knox users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ knox_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ knox_local_accounts }}" diff --git a/roles/prereq_knox/tasks/main.yml b/roles/prereq_knox/tasks/main.yml new file mode 100644 index 00000000..07932309 --- /dev/null +++ b/roles/prereq_knox/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ knox_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ knox_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_knox/vars/main.yml b/roles/prereq_knox/vars/main.yml new file mode 100644 index 00000000..3e963a1f --- /dev/null +++ b/roles/prereq_knox/vars/main.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +knox_local_accounts: + - user: knox + home: /var/lib/knox + comment: Knox + keystore_acl: true + extra_groups: [hadoop] From 4c3a7366d06b27018168c4a937b60e9c48aa4ce0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:02 -0400 Subject: [PATCH 30/72] Add Apache Knox database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_knox_database/README.md | 79 ++++ roles/prereq_knox_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_knox_database/tasks/main.yml | 24 ++ 11 files changed, 853 insertions(+) create mode 100644 roles/prereq_knox_database/README.md create mode 100644 roles/prereq_knox_database/defaults/main.yml create mode 100644 roles/prereq_knox_database/meta/argument_specs.yml create mode 100644 roles/prereq_knox_database/molecule/default/converge.yml create mode 100644 roles/prereq_knox_database/molecule/default/create.yml create mode 100644 roles/prereq_knox_database/molecule/default/destroy.yml create mode 100644 roles/prereq_knox_database/molecule/default/molecule.yml create mode 100644 roles/prereq_knox_database/molecule/default/prepare.yml create mode 100644 roles/prereq_knox_database/molecule/default/requirements.yml create mode 100644 roles/prereq_knox_database/molecule/default/verify.yml create mode 100644 roles/prereq_knox_database/tasks/main.yml diff --git a/roles/prereq_knox_database/README.md b/roles/prereq_knox_database/README.md new file mode 100644 index 00000000..b067a94a --- /dev/null +++ b/roles/prereq_knox_database/README.md @@ -0,0 +1,79 @@ +# prereq_knox_database + +Set up database and user accounts for Knox + +This role automates the setup of a database and its associated user accounts specifically for Apache Knox services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `knox_database`. +- Create a new database user specified by `knox_username` with the password from `knox_password`. +- Grant ownership and all necessary privileges to the `knox_username` for the new database. +- Ensure the database is configured correctly for Knox operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `knox_username` | `str` | `False` | `knox` | The username for the Knox database user. This user will also be the owner of the database. | +| `knox_password` | `str` | `False` | `knox` | The password for the Knox database user. It is highly recommended to override this default in production. | +| `knox_database` | `str` | `False` | `knox` | The name of the database to be created for Knox. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Knox database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_knox_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Knox database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_knox_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + knox_username: "my_knox_user" + knox_password: "a_strong_knox_password" + knox_database: "my_knox_db" +``` + +# License + +``` +Copyright 2025 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_knox_database/defaults/main.yml b/roles/prereq_knox_database/defaults/main.yml new file mode 100644 index 00000000..bc376b7b --- /dev/null +++ b/roles/prereq_knox_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +knox_username: knox +knox_password: knox +knox_database: knox diff --git a/roles/prereq_knox_database/meta/argument_specs.yml b/roles/prereq_knox_database/meta/argument_specs.yml new file mode 100644 index 00000000..84fd9b2a --- /dev/null +++ b/roles/prereq_knox_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Knox + description: + - Set up the Apache Knox database and its associated user accounts, ensuring proper configuration for Knox operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `knox_username`, `knox_password`, and + `knox_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + knox_username: + description: The username for the Knox database user and owner of the database. + type: str + required: false + default: knox + knox_password: + description: The password for the Knox database user. + type: str + required: false + default: knox + knox_database: + description: The name of the database to be created for Knox. + type: str + required: false + default: knox diff --git a/roles/prereq_knox_database/molecule/default/converge.yml b/roles/prereq_knox_database/molecule/default/converge.yml new file mode 100644 index 00000000..2b26fff9 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Knox database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_knox_database diff --git a/roles/prereq_knox_database/molecule/default/create.yml b/roles/prereq_knox_database/molecule/default/create.yml new file mode 100644 index 00000000..e40c7f7a --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_knox_database/molecule/default/destroy.yml b/roles/prereq_knox_database/molecule/default/destroy.yml new file mode 100644 index 00000000..8d3a4863 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_knox_database/molecule/default/molecule.yml b/roles/prereq_knox_database/molecule/default/molecule.yml new file mode 100644 index 00000000..71a7da12 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_knox_database-rhel9-4 + Project: Molecule testing for prereq_knox_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_knox_database/molecule/default/prepare.yml b/roles/prereq_knox_database/molecule/default/prepare.yml new file mode 100644 index 00000000..a33a4190 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_knox_database/molecule/default/requirements.yml b/roles/prereq_knox_database/molecule/default/requirements.yml new file mode 100644 index 00000000..3a5c5a35 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_knox_database/molecule/default/verify.yml b/roles/prereq_knox_database/molecule/default/verify.yml new file mode 100644 index 00000000..a98fdce1 --- /dev/null +++ b/roles/prereq_knox_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'knox';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database knox does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'knox';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User knox does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_knox_database/tasks/main.yml b/roles/prereq_knox_database/tasks/main.yml new file mode 100644 index 00000000..489dfe95 --- /dev/null +++ b/roles/prereq_knox_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ knox_details }}" + knox_details: + - user: "{{ knox_username }}" + password: "{{ knox_password }}" + db: "{{ knox_database }}" + no_log: true From e5c6dc455bf4d87f0a76501027e26aec0e5d9880 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:03 -0400 Subject: [PATCH 31/72] Add Apache Kudu prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_kudu/README.md | 52 +++ roles/prereq_kudu/meta/argument_specs.yml | 22 ++ .../prereq_kudu/molecule/default/converge.yml | 23 ++ roles/prereq_kudu/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_kudu/molecule/default/destroy.yml | 157 ++++++++ .../prereq_kudu/molecule/default/molecule.yml | 49 +++ .../prereq_kudu/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_kudu/molecule/default/verify.yml | 36 ++ roles/prereq_kudu/tasks/main.yml | 39 ++ roles/prereq_kudu/vars/main.yml | 21 ++ 11 files changed, 794 insertions(+) create mode 100644 roles/prereq_kudu/README.md create mode 100644 roles/prereq_kudu/meta/argument_specs.yml create mode 100644 roles/prereq_kudu/molecule/default/converge.yml create mode 100644 roles/prereq_kudu/molecule/default/create.yml create mode 100644 roles/prereq_kudu/molecule/default/destroy.yml create mode 100644 roles/prereq_kudu/molecule/default/molecule.yml create mode 100644 roles/prereq_kudu/molecule/default/prepare.yml create mode 100644 roles/prereq_kudu/molecule/default/requirements.yml create mode 100644 roles/prereq_kudu/molecule/default/verify.yml create mode 100644 roles/prereq_kudu/tasks/main.yml create mode 100644 roles/prereq_kudu/vars/main.yml diff --git a/roles/prereq_kudu/README.md b/roles/prereq_kudu/README.md new file mode 100644 index 00000000..41153730 --- /dev/null +++ b/roles/prereq_kudu/README.md @@ -0,0 +1,52 @@ +# prereq_kudu + +Set up for Kudu + +This role prepares a host for Apache Kudu usage by creating a dedicated system user and group named `kudu`. This user is essential for running Kudu processes with appropriate permissions and isolation. + +The role will: +- Create the `kudu` system user and group. +- Configure home directories and other necessary local paths for the `kudu` user, if required. +- Ensure appropriate permissions are set for files and directories related to Kudu. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: kudu_nodes + tasks: + - name: Set up the kudu user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_kudu +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_kudu/meta/argument_specs.yml b/roles/prereq_kudu/meta/argument_specs.yml new file mode 100644 index 00000000..543cc78f --- /dev/null +++ b/roles/prereq_kudu/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Kudu + description: | + Set up for Apache Kudu usage, notably, create the local C(kudu) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_kudu/molecule/default/converge.yml b/roles/prereq_kudu/molecule/default/converge.yml new file mode 100644 index 00000000..85b4f9f5 --- /dev/null +++ b/roles/prereq_kudu/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Kudu + ansible.builtin.import_role: + name: cloudera.exe.prereq_kudu diff --git a/roles/prereq_kudu/molecule/default/create.yml b/roles/prereq_kudu/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_kudu/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_kudu/molecule/default/destroy.yml b/roles/prereq_kudu/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_kudu/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_kudu/molecule/default/molecule.yml b/roles/prereq_kudu/molecule/default/molecule.yml new file mode 100644 index 00000000..690dca96 --- /dev/null +++ b/roles/prereq_kudu/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_kudu-rhel9-4 + Project: Molecule testing for prereq_kudu +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_kudu/molecule/default/prepare.yml b/roles/prereq_kudu/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_kudu/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_kudu/molecule/default/requirements.yml b/roles/prereq_kudu/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_kudu/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_kudu/molecule/default/verify.yml b/roles/prereq_kudu/molecule/default/verify.yml new file mode 100644 index 00000000..38380ea3 --- /dev/null +++ b/roles/prereq_kudu/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for kudu users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ kudu_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ kudu_local_accounts }}" diff --git a/roles/prereq_kudu/tasks/main.yml b/roles/prereq_kudu/tasks/main.yml new file mode 100644 index 00000000..9597f4a7 --- /dev/null +++ b/roles/prereq_kudu/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ kudu_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ kudu_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_kudu/vars/main.yml b/roles/prereq_kudu/vars/main.yml new file mode 100644 index 00000000..94d5e7f6 --- /dev/null +++ b/roles/prereq_kudu/vars/main.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kudu_local_accounts: + - user: kudu + home: /var/lib/kudu + comment: Kudu + key_acl: true + key_password_acl: true From 006fe65beffb0e44d3e203712ead6f939a43bb56 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:04 -0400 Subject: [PATCH 32/72] Add Apache Livy prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_livy/README.md | 52 +++ roles/prereq_livy/meta/argument_specs.yml | 22 ++ .../prereq_livy/molecule/default/converge.yml | 23 ++ roles/prereq_livy/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_livy/molecule/default/destroy.yml | 157 ++++++++ .../prereq_livy/molecule/default/molecule.yml | 49 +++ .../prereq_livy/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_livy/molecule/default/verify.yml | 36 ++ roles/prereq_livy/tasks/main.yml | 39 ++ roles/prereq_livy/vars/main.yml | 20 ++ 11 files changed, 793 insertions(+) create mode 100644 roles/prereq_livy/README.md create mode 100644 roles/prereq_livy/meta/argument_specs.yml create mode 100644 roles/prereq_livy/molecule/default/converge.yml create mode 100644 roles/prereq_livy/molecule/default/create.yml create mode 100644 roles/prereq_livy/molecule/default/destroy.yml create mode 100644 roles/prereq_livy/molecule/default/molecule.yml create mode 100644 roles/prereq_livy/molecule/default/prepare.yml create mode 100644 roles/prereq_livy/molecule/default/requirements.yml create mode 100644 roles/prereq_livy/molecule/default/verify.yml create mode 100644 roles/prereq_livy/tasks/main.yml create mode 100644 roles/prereq_livy/vars/main.yml diff --git a/roles/prereq_livy/README.md b/roles/prereq_livy/README.md new file mode 100644 index 00000000..19e64b23 --- /dev/null +++ b/roles/prereq_livy/README.md @@ -0,0 +1,52 @@ +# prereq_livy + +Set up for Livy + +This role prepares a host for Apache Livy usage by creating a dedicated system user and group named `livy`. This user is essential for running Livy processes with appropriate permissions and isolation. + +The role will: +- Create the `livy` system user and group. +- Configure home directories and other necessary local paths for the `livy` user, if required. +- Ensure appropriate permissions are set for files and directories related to Livy. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: livy_nodes + tasks: + - name: Set up the livy user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_livy +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_livy/meta/argument_specs.yml b/roles/prereq_livy/meta/argument_specs.yml new file mode 100644 index 00000000..5fd8be89 --- /dev/null +++ b/roles/prereq_livy/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Livy + description: | + Set up for Apache Livy usage, notably, create the local C(livy) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_livy/molecule/default/converge.yml b/roles/prereq_livy/molecule/default/converge.yml new file mode 100644 index 00000000..bba60d38 --- /dev/null +++ b/roles/prereq_livy/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Livy + ansible.builtin.import_role: + name: cloudera.exe.prereq_livy diff --git a/roles/prereq_livy/molecule/default/create.yml b/roles/prereq_livy/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_livy/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_livy/molecule/default/destroy.yml b/roles/prereq_livy/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_livy/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_livy/molecule/default/molecule.yml b/roles/prereq_livy/molecule/default/molecule.yml new file mode 100644 index 00000000..82aa9383 --- /dev/null +++ b/roles/prereq_livy/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_livy-rhel9-4 + Project: Molecule testing for prereq_livy +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_livy/molecule/default/prepare.yml b/roles/prereq_livy/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_livy/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_livy/molecule/default/requirements.yml b/roles/prereq_livy/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_livy/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_livy/molecule/default/verify.yml b/roles/prereq_livy/molecule/default/verify.yml new file mode 100644 index 00000000..2441673b --- /dev/null +++ b/roles/prereq_livy/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for livy users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ livy_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ livy_local_accounts }}" diff --git a/roles/prereq_livy/tasks/main.yml b/roles/prereq_livy/tasks/main.yml new file mode 100644 index 00000000..873c1910 --- /dev/null +++ b/roles/prereq_livy/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ livy_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ livy_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_livy/vars/main.yml b/roles/prereq_livy/vars/main.yml new file mode 100644 index 00000000..bf700116 --- /dev/null +++ b/roles/prereq_livy/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +livy_local_accounts: + - user: livy + home: /var/lib/livy + comment: Livy + keystore_acl: true From cde50efc41a385fe0141778b1624330906eb6ec2 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:04 -0400 Subject: [PATCH 33/72] Add local accounts prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_local_account/README.md | 77 ++++ roles/prereq_local_account/defaults/main.yml | 16 + .../meta/argument_specs.yml | 50 +++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 23 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 43 +++ roles/prereq_local_account/tasks/main.yml | 39 ++ roles/prereq_local_account/vars/main.yml | 17 + 12 files changed, 851 insertions(+) create mode 100644 roles/prereq_local_account/README.md create mode 100644 roles/prereq_local_account/defaults/main.yml create mode 100644 roles/prereq_local_account/meta/argument_specs.yml create mode 100644 roles/prereq_local_account/molecule/default/converge.yml create mode 100644 roles/prereq_local_account/molecule/default/create.yml create mode 100644 roles/prereq_local_account/molecule/default/destroy.yml create mode 100644 roles/prereq_local_account/molecule/default/molecule.yml create mode 100644 roles/prereq_local_account/molecule/default/prepare.yml create mode 100644 roles/prereq_local_account/molecule/default/requirements.yml create mode 100644 roles/prereq_local_account/molecule/default/verify.yml create mode 100644 roles/prereq_local_account/tasks/main.yml create mode 100644 roles/prereq_local_account/vars/main.yml diff --git a/roles/prereq_local_account/README.md b/roles/prereq_local_account/README.md new file mode 100644 index 00000000..522ca448 --- /dev/null +++ b/roles/prereq_local_account/README.md @@ -0,0 +1,77 @@ +# prereq_local_account + +Set up local user accounts + +This role automates the creation and management of local user accounts on a host. It can create multiple user accounts with specified home directories, UID, shell, comments, and group memberships. The role ensures that the user's home directory is created with the correct permissions. + +The role will: +- Iterate through the list of `local_accounts` provided. +- For each account, create the system user and their primary group. +- Create the user's home directory with the specified permissions. +- Set the user's UID, shell, and comment if provided. +- Add the user to any specified `extra_groups`. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `local_accounts` | `list` of `dict` | `False` | `[]` | A list of user accounts to create. Each item in the list is a dictionary with the following keys. | +|     `user` | `str` | `True` | | User account name. | +|     `home` | `path` | `True` | | User account home directory. The role will create this directory. | +|     `uid` | `int` | `False` | | User account UID (User ID). | +|     `shell` | `str` | `False` | `/sbin/nologin` | User account shell. | +|     `comment` | `str` | `False` | | Comments for the user account entry. | +|     `extra_groups` | `list` of `str` | `False` | `[]` | Additional groups to assign (append) to the user account. | +|     `mode` | `str` | `False` | `0755` | Permissions for the user account's home directory. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Create various local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: + - user: "appuser" + home: "/home/appuser" + - user: "jenkins" + home: "/var/lib/jenkins" + uid: 1001 + shell: "/bin/bash" + comment: "CI/CD User" + extra_groups: + - "sudo" + - "docker" + - user: "monitoring" + home: "/home/monitoring" + mode: "0700" + shell: "/bin/false" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_local_account/defaults/main.yml b/roles/prereq_local_account/defaults/main.yml new file mode 100644 index 00000000..e52b0327 --- /dev/null +++ b/roles/prereq_local_account/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +local_accounts: [] diff --git a/roles/prereq_local_account/meta/argument_specs.yml b/roles/prereq_local_account/meta/argument_specs.yml new file mode 100644 index 00000000..25c699f0 --- /dev/null +++ b/roles/prereq_local_account/meta/argument_specs.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up local user accounts + description: | + Set up local user accounts and create the user's C(HOME) directory. + author: Cloudera Labs + options: + local_accounts: + description: A list of user accounts. + type: list + elements: dict + default: [] + options: + user: + description: User account name. + required: true + home: + description: User account C(HOME) directory. + type: path + required: true + uid: + description: User account C(UID). + type: int + shell: + description: User account C(SHELL). + default: "/sbin/nologin" + comment: + description: Comments for the user account entry. + extra_groups: + description: Additional groups to assign (append) to the user account. + type: list + elements: str + mode: + description: Permissions for user account C(HOME) directory. + default: "0755" diff --git a/roles/prereq_local_account/molecule/default/converge.yml b/roles/prereq_local_account/molecule/default/converge.yml new file mode 100644 index 00000000..2854d5c3 --- /dev/null +++ b/roles/prereq_local_account/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account diff --git a/roles/prereq_local_account/molecule/default/create.yml b/roles/prereq_local_account/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_local_account/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_local_account/molecule/default/destroy.yml b/roles/prereq_local_account/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_local_account/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_local_account/molecule/default/molecule.yml b/roles/prereq_local_account/molecule/default/molecule.yml new file mode 100644 index 00000000..61a47d7b --- /dev/null +++ b/roles/prereq_local_account/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-cloudera.exe.prereq_local_account-rhel9-4 + Project: Molecule testing for cloudera.exe.prereq_local_account +provisioner: + name: ansible + inventory: + group_vars: + all: + local_accounts: + - user: test + home: /var/lib/test + comment: Test user account + extra_groups: + - example + mode: "0700" diff --git a/roles/prereq_local_account/molecule/default/prepare.yml b/roles/prereq_local_account/molecule/default/prepare.yml new file mode 100644 index 00000000..22923220 --- /dev/null +++ b/roles/prereq_local_account/molecule/default/prepare.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create extra test group + ansible.builtin.group: + name: "example" diff --git a/roles/prereq_local_account/molecule/default/requirements.yml b/roles/prereq_local_account/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_local_account/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_local_account/molecule/default/verify.yml b/roles/prereq_local_account/molecule/default/verify.yml new file mode 100644 index 00000000..e3620f99 --- /dev/null +++ b/roles/prereq_local_account/molecule/default/verify.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check for test user + ansible.builtin.command: grep test /etc/passwd + register: __user + failed_when: __user.rc != 0 + changed_when: false + + - name: Check for test user group membership + ansible.builtin.command: groups test + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("example") + changed_when: false + + - name: Stat test user home directory + ansible.builtin.stat: + path: "{{ local_accounts[0].home }}" + register: __home + + - name: Check test user home directory + ansible.builtin.assert: + that: + - __home.stat.exists + - __home.stat.mode == "0700" + - __home.stat.pw_name == "test" + - __home.stat.gr_name == "test" diff --git a/roles/prereq_local_account/tasks/main.yml b/roles/prereq_local_account/tasks/main.yml new file mode 100644 index 00000000..98dcd6c8 --- /dev/null +++ b/roles/prereq_local_account/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.user: + name: "{{ account.user }}" + home: "{{ account.home }}" + uid: "{{ account.uid | default(omit) }}" + shell: "{{ account.shell | default(local_account_shell) }}" + comment: "{{ account.comment | default(omit) }}" + groups: "{{ account.extra_groups | default(omit) }}" + append: "{{ (account.extra_groups is defined) | ternary('yes', omit) }}" + loop: "{{ local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Set local user home directory permissions + ansible.builtin.file: + path: "{{ account.home }}" + owner: "{{ account.user }}" + group: "{{ account.user }}" + mode: "{{ account.mode | default(local_account_mode) }}" + loop: "{{ local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.home }}" diff --git a/roles/prereq_local_account/vars/main.yml b/roles/prereq_local_account/vars/main.yml new file mode 100644 index 00000000..503bb951 --- /dev/null +++ b/roles/prereq_local_account/vars/main.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +local_account_shell: /sbin/nologin +local_account_mode: "0755" From 21075785597110c6841954c67077f2e26093b4bd Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:05 -0400 Subject: [PATCH 34/72] Add MapReduce prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_mapreduce/README.md | 53 +++ .../prereq_mapreduce/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 38 ++ roles/prereq_mapreduce/tasks/main.yml | 38 ++ roles/prereq_mapreduce/vars/main.yml | 20 ++ 11 files changed, 796 insertions(+) create mode 100644 roles/prereq_mapreduce/README.md create mode 100644 roles/prereq_mapreduce/meta/argument_specs.yml create mode 100644 roles/prereq_mapreduce/molecule/default/converge.yml create mode 100644 roles/prereq_mapreduce/molecule/default/create.yml create mode 100644 roles/prereq_mapreduce/molecule/default/destroy.yml create mode 100644 roles/prereq_mapreduce/molecule/default/molecule.yml create mode 100644 roles/prereq_mapreduce/molecule/default/prepare.yml create mode 100644 roles/prereq_mapreduce/molecule/default/requirements.yml create mode 100644 roles/prereq_mapreduce/molecule/default/verify.yml create mode 100644 roles/prereq_mapreduce/tasks/main.yml create mode 100644 roles/prereq_mapreduce/vars/main.yml diff --git a/roles/prereq_mapreduce/README.md b/roles/prereq_mapreduce/README.md new file mode 100644 index 00000000..72752d71 --- /dev/null +++ b/roles/prereq_mapreduce/README.md @@ -0,0 +1,53 @@ +# prereq_mapreduce + +Set up for MapReduce + +This role prepares a host for MapReduce usage by creating a dedicated system user and group named `mapred`. This user is essential for running MapReduce processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure MapReduce communication. + +The role will: +- Create the `mapred` system user and group. +- Configure home directories and other necessary local paths for the `mapred` user, if required. +- Ensure appropriate permissions are set for files and directories related to MapReduce. +- Configure TLS ACLs to secure MapReduce communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: mapreduce_nodes + tasks: + - name: Set up the mapred user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_mapreduce +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_mapreduce/meta/argument_specs.yml b/roles/prereq_mapreduce/meta/argument_specs.yml new file mode 100644 index 00000000..d52f96a8 --- /dev/null +++ b/roles/prereq_mapreduce/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for MapReduce + description: | + Set up for MapReduce usage, notably, create the local C(mapred) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_mapreduce/molecule/default/converge.yml b/roles/prereq_mapreduce/molecule/default/converge.yml new file mode 100644 index 00000000..f70c3a3c --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Map Reduce + ansible.builtin.import_role: + name: cloudera.exe.prereq_mapreduce diff --git a/roles/prereq_mapreduce/molecule/default/create.yml b/roles/prereq_mapreduce/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_mapreduce/molecule/default/destroy.yml b/roles/prereq_mapreduce/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_mapreduce/molecule/default/molecule.yml b/roles/prereq_mapreduce/molecule/default/molecule.yml new file mode 100644 index 00000000..176a8518 --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_mapreduce-rhel9-4 + Project: Molecule testing for prereq_mapreduce +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_mapreduce/molecule/default/prepare.yml b/roles/prereq_mapreduce/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_mapreduce/molecule/default/requirements.yml b/roles/prereq_mapreduce/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_mapreduce/molecule/default/verify.yml b/roles/prereq_mapreduce/molecule/default/verify.yml new file mode 100644 index 00000000..496b6243 --- /dev/null +++ b/roles/prereq_mapreduce/molecule/default/verify.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for hadoop group + ansible.builtin.command: grep hadoop /etc/group + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for mapred user + ansible.builtin.command: grep mapred /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ mapreduce_local_accounts }}" diff --git a/roles/prereq_mapreduce/tasks/main.yml b/roles/prereq_mapreduce/tasks/main.yml new file mode 100644 index 00000000..0b37288a --- /dev/null +++ b/roles/prereq_mapreduce/tasks/main.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ mapreduce_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ mapreduce_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_mapreduce/vars/main.yml b/roles/prereq_mapreduce/vars/main.yml new file mode 100644 index 00000000..468c10b1 --- /dev/null +++ b/roles/prereq_mapreduce/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mapreduce_local_accounts: + - user: mapred + home: /var/lib/hadoop-mapreduce + comment: MapReduce + extra_groups: [hadoop] From fc5022429587a7c3b5b99b9a40ebea455de9079c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:06 -0400 Subject: [PATCH 35/72] Add DNS networking prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_network_dns/README.md | 67 ++++ roles/prereq_network_dns/defaults/main.yml | 22 ++ roles/prereq_network_dns/handlers/main.yml | 28 ++ .../meta/argument_specs.yml | 50 +++ .../molecule/default/converge.yml | 46 +++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 45 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 30 ++ roles/prereq_network_dns/tasks/dhcp.yml | 29 ++ roles/prereq_network_dns/tasks/main.yml | 87 +++++ roles/prereq_network_dns/tasks/netplan.yml | 42 +++ .../tasks/network-manager.yml | 24 ++ .../templates/netplan.yaml.j2 | 7 + .../templates/networkmanager.conf.j2 | 8 + .../templates/resolv.conf.j2 | 3 + roles/prereq_network_dns/vars/default.yml | 13 + 18 files changed, 1015 insertions(+) create mode 100644 roles/prereq_network_dns/README.md create mode 100644 roles/prereq_network_dns/defaults/main.yml create mode 100644 roles/prereq_network_dns/handlers/main.yml create mode 100644 roles/prereq_network_dns/meta/argument_specs.yml create mode 100644 roles/prereq_network_dns/molecule/default/converge.yml create mode 100644 roles/prereq_network_dns/molecule/default/create.yml create mode 100644 roles/prereq_network_dns/molecule/default/destroy.yml create mode 100644 roles/prereq_network_dns/molecule/default/molecule.yml create mode 100644 roles/prereq_network_dns/molecule/default/requirements.yml create mode 100644 roles/prereq_network_dns/molecule/default/verify.yml create mode 100644 roles/prereq_network_dns/tasks/dhcp.yml create mode 100644 roles/prereq_network_dns/tasks/main.yml create mode 100644 roles/prereq_network_dns/tasks/netplan.yml create mode 100644 roles/prereq_network_dns/tasks/network-manager.yml create mode 100644 roles/prereq_network_dns/templates/netplan.yaml.j2 create mode 100644 roles/prereq_network_dns/templates/networkmanager.conf.j2 create mode 100644 roles/prereq_network_dns/templates/resolv.conf.j2 create mode 100644 roles/prereq_network_dns/vars/default.yml diff --git a/roles/prereq_network_dns/README.md b/roles/prereq_network_dns/README.md new file mode 100644 index 00000000..118c1c03 --- /dev/null +++ b/roles/prereq_network_dns/README.md @@ -0,0 +1,67 @@ +# prereq_network_dns + +Set up hostname and DNS networking + +This role automates the configuration of a host's networking settings, including its hostname and DNS. It is designed to work with various system-level network configuration tools like `cloud-init`, `NetworkManager`, `Netplan`, and `dhclient`. The role is crucial for ensuring proper name resolution and consistent IP address configuration within a cluster or managed environment. + +The role will: +- Set the hostname of the target host. +- Configure DNS settings, including the search domain (`network_dns_domain`). +- Configure a list of DNS forwarders for name resolution (`network_dns_forwarders`). +- Apply these configurations to the appropriate network management system (`cloud-init`, `dhclient`, `netplan`, etc.) on the host. +- Set a static IP address or ensure a specific IP is used as the default for the host. + +# Requirements + +- Root or `sudo` privileges are required on the target host to modify system and network configuration files. +- This role assumes the presence of one of the targeted network configuration systems on the host (e.g., `NetworkManager` on Red Hat-based systems or `Netplan` on Ubuntu). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `network_cloud_init_path` | `path` | `False` | `/etc/cloud/cloud.cfg` | Path to the `cloud-init` configuration file. The role will manage this file if it exists. | +| `network_dhclient_path` | `path` | `False` | `/etc/dhcp/dhclient.conf` | Path to the DHCP client configuration file. | +| `network_netplan_dir` | `path` | `False` | `/etc/netplan` | Path to the Netplan configuration directory. | +| `network_dns_domain` | `str` | `True` | | The DNS search domain for the host (e.g., `example.internal`). | +| `network_dns_forwarders` | `list` of `str` | `True` | | A prioritized list of DNS name server IP addresses to be used for name resolution. | +| `network_ip_address` | `str` | `True` | | The IP address of the host that the role should configure as the default. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Set up network for a cluster node + ansible.builtin.import_role: + name: cloudera.exe.prereq_network_dns + vars: + network_dns_domain: "example.internal" + network_dns_forwarders: + - "10.0.0.10" + - "8.8.8.8" + network_ip_address: "10.0.1.100" + # The other optional path variables will use their defaults and will be managed if those files exist. +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_network_dns/defaults/main.yml b/roles/prereq_network_dns/defaults/main.yml new file mode 100644 index 00000000..31c59dd4 --- /dev/null +++ b/roles/prereq_network_dns/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +network_cloud_init_path: /etc/cloud/cloud.cfg +network_dhclient_path: /etc/dhcp/dhclient.conf +network_netplan_dir: /etc/netplan + +network_ip_address: "{{ undef(hint='Please define the DNS IP address for the host') }}" +network_dns_domain: "{{ undef(hint='Please define the DNS search domain') }}" +network_dns_forwarders: "{{ undef(hint='Please define the DNS forwarders') }}" diff --git a/roles/prereq_network_dns/handlers/main.yml b/roles/prereq_network_dns/handlers/main.yml new file mode 100644 index 00000000..aa1f3a3a --- /dev/null +++ b/roles/prereq_network_dns/handlers/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Restart NetworkManager + ansible.builtin.service: + name: NetworkManager + state: restarted + +- name: Apply netplan configuration + ansible.builtin.command: netplan apply + changed_when: false + +- name: Restart systemd-resolved + ansible.builtin.service: + name: systemd-resolved + state: restarted diff --git a/roles/prereq_network_dns/meta/argument_specs.yml b/roles/prereq_network_dns/meta/argument_specs.yml new file mode 100644 index 00000000..989d0c00 --- /dev/null +++ b/roles/prereq_network_dns/meta/argument_specs.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up hostname and DNS networking + description: | + Set up hostname and DNS networking, targeting various systems that control such configuration, such as C(cloud-init) and C(NetworkManager). + Set up DNS forwarders for name resolution. + author: Cloudera Labs + options: + network_cloud_init_path: + description: + - Path to the C(cloud-init) configuration file. + type: path + default: /etc/cloud/cloud.cfg + network_dhclient_path: + description: + - Path to the DHCP client configuration file. + type: path + default: /etc/dhcp/dhclient.conf + network_netplan_dir: + description: + - Path to the Netplan configuration directory. + type: path + default: /etc/netplan + network_dns_domain: + description: + - DNS search domain. + required: true + network_dns_forwarders: + description: + - List of DNS name servers, in preferred order. + required: true + network_ip_address: + description: + - IP address of the host to set as default. + required: true diff --git a/roles/prereq_network_dns/molecule/default/converge.yml b/roles/prereq_network_dns/molecule/default/converge.yml new file mode 100644 index 00000000..9c418a60 --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/converge.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Extract the VPC subnet ID from the Molecule platform configuration + ansible.builtin.set_fact: + test_subnet_id: "{{ molecule_yml.platforms | selectattr('name', 'eq', inventory_hostname) | map(attribute='vpc_subnet_id') | first }}" + + - name: Retrieve the VPC subnet details + amazon.aws.ec2_vpc_subnet_info: + subnet_id: "{{ test_subnet_id }}" + register: __subnet + become: false + delegate_to: localhost + + - name: Retrieve the VPC details + amazon.aws.ec2_vpc_net_info: + vpc_ids: "{{ __subnet.subnets | map(attribute='vpc_id') | first }}" + register: __vpc + become: false + delegate_to: localhost + + - name: Set up host and DNS networking + ansible.builtin.import_role: + name: cloudera.exe.prereq_network_dns + vars: + # See https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html + vpc_cidr: "{{ __vpc.vpcs | map(attribute='cidr_block') | first }}" + network_dns_forwarders: ["{{ vpc_cidr | ansible.utils.ipmath(2) }}"] + network_ip_address: "{{ ansible_default_ipv4.address }}" diff --git a/roles/prereq_network_dns/molecule/default/create.yml b/roles/prereq_network_dns/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_network_dns/molecule/default/destroy.yml b/roles/prereq_network_dns/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_network_dns/molecule/default/molecule.yml b/roles/prereq_network_dns/molecule/default/molecule.yml new file mode 100644 index 00000000..737eb855 --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/molecule.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_network_dns-rhel9-4 + Project: Molecule testing for prereq_network_dns +provisioner: + name: ansible + inventory: + group_vars: + all: + network_dns_domain: molecule.internal + # Note, network_dns_forwarders is set in converge.yml + # due to runtime dependencies on VPC CIDR diff --git a/roles/prereq_network_dns/molecule/default/requirements.yml b/roles/prereq_network_dns/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_network_dns/molecule/default/verify.yml b/roles/prereq_network_dns/molecule/default/verify.yml new file mode 100644 index 00000000..6acbd823 --- /dev/null +++ b/roles/prereq_network_dns/molecule/default/verify.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: true + tasks: + - name: Check name resolution + ansible.builtin.command: hostname + register: __hostname + failed_when: __hostname.stdout != inventory_hostname + changed_when: false + + - name: Check external name resolution + ansible.builtin.command: ping -c 1 cloudera.com + register: __ping + failed_when: __ping.rc != 0 or __ping.stdout is search("Name or service not known") + changed_when: false diff --git a/roles/prereq_network_dns/tasks/dhcp.yml b/roles/prereq_network_dns/tasks/dhcp.yml new file mode 100644 index 00000000..98310b2a --- /dev/null +++ b/roles/prereq_network_dns/tasks/dhcp.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set dhclient.conf for domain search and name servers + ansible.builtin.lineinfile: + path: "{{ network_dhclient_path }}" + regex: "^(#)?{{ dhclient_entry.value }}" + line: "{{ dhclient_entry.value }}" + state: present + loop: "{{ entries | dict2items }}" + loop_control: + loop_var: dhclient_entry + label: "{{ dhclient_entry.key }}" + vars: + entries: + domain_search: supersede domain-search "{{ network_dns_domain }}"; + domain_name_servers: supersede domain-name-servers {{ network_dns_forwarders | join(', ') }}; diff --git a/roles/prereq_network_dns/tasks/main.yml b/roles/prereq_network_dns/tasks/main.yml new file mode 100644 index 00000000..974b288e --- /dev/null +++ b/roles/prereq_network_dns/tasks/main.yml @@ -0,0 +1,87 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Check for cloud-init + ansible.builtin.stat: + path: "{{ network_cloud_init_path }}" + register: __cloud_init + +- name: Set cloud-init to preserve hostname + when: __cloud_init.stat.exists + ansible.builtin.lineinfile: + path: "{{ network_cloud_init_path }}" + regex: "^(#)?preserve_hostname" + line: "preserve_hostname: true" + state: present + +- name: Set hostname to Ansible FQDN inventory hostname + ansible.builtin.hostname: + name: "{{ inventory_hostname }}" + +- name: Set Ansible FQDN inventory hostname and IP address in /etc/hosts + ansible.builtin.lineinfile: + path: /etc/hosts + line: "{{ network_ip_address }} {{ inventory_hostname }} {{ inventory_hostname_short }}" + regexp: "[\\d\\.]+\\s+{{ inventory_hostname }} {{ inventory_hostname_short }}" + state: present + backup: true + +- name: Gather service details + ansible.builtin.service_facts: + +- name: Configure NetworkManager + when: "'NetworkManager.service' in ansible_facts.services and ansible_facts.services['NetworkManager.service']['state'] == 'running'" + ansible.builtin.include_tasks: network-manager.yml + register: __network_manager + +# Netplan can also use NetworkManager, so the logic will need to change +- name: Configure systemd-resolved via netplan + when: "'systemd-resolved.service' in ansible_facts.services and ansible_facts.services['systemd-resolved.service']['state'] == 'running'" + ansible.builtin.include_tasks: netplan.yml + register: __netplan + +- name: Handle DHCP + when: __network_manager is skipped and __netplan is skipped + block: + - name: Check for existence of DHCP client + ansible.builtin.stat: + path: "{{ network_dhclient_path }}" + register: __dhclient_conf + + - name: Configure DHCP client + when: __dhclient_conf.stat.exists + ansible.builtin.include_tasks: dhcp.yml + register: __dhcp + +- name: Configure /etc/resolv.conf directly + when: __network_manager is skipped and __netplan is skipped and __dhcp is skipped + ansible.builtin.template: + src: resolv.conf.j2 + dest: /etc/resolv.conf + mode: "0644" + +- name: Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/roles/prereq_network_dns/tasks/netplan.yml b/roles/prereq_network_dns/tasks/netplan.yml new file mode 100644 index 00000000..27cba3da --- /dev/null +++ b/roles/prereq_network_dns/tasks/netplan.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather networking details + ansible.builtin.setup: + gather_subset: network + +- name: Set netplan configuration fragment for DNS + ansible.builtin.template: + dest: "{{ [network_netplan_dir, '99-cldr-internal.yaml'] | path_join }}" + src: netplan.yaml.j2 + mode: "0644" + owner: root + group: root + vars: + internal_interface: >- + {{ + ['ansible_'] | + product(ansible_interfaces) | + map('join') | + map('extract', hostvars[inventory_hostname]) | + selectattr('type', 'eq', 'ether') | + selectattr('ipv4.address', 'eq', network_ip_address) | + map(attribute='device') | + first + }} + register: __netplan_template + notify: + - Apply netplan configuration + - Restart systemd-resolved diff --git a/roles/prereq_network_dns/tasks/network-manager.yml b/roles/prereq_network_dns/tasks/network-manager.yml new file mode 100644 index 00000000..a1864994 --- /dev/null +++ b/roles/prereq_network_dns/tasks/network-manager.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://access.redhat.com/solutions/3078301 +- name: Set NetworkManager configuration fragment for DNS + ansible.builtin.template: + dest: /etc/NetworkManager/conf.d/cldr-dns.conf + src: networkmanager.conf.j2 + mode: "0644" + owner: root + group: root + notify: Restart NetworkManager diff --git a/roles/prereq_network_dns/templates/netplan.yaml.j2 b/roles/prereq_network_dns/templates/netplan.yaml.j2 new file mode 100644 index 00000000..264484e8 --- /dev/null +++ b/roles/prereq_network_dns/templates/netplan.yaml.j2 @@ -0,0 +1,7 @@ +network: + ethernets: + {{ internal_interface }}: + nameservers: + addresses: {{ network_dns_forwarders }} + search: [{{ network_dns_domain }}] + version: 2 diff --git a/roles/prereq_network_dns/templates/networkmanager.conf.j2 b/roles/prereq_network_dns/templates/networkmanager.conf.j2 new file mode 100644 index 00000000..1e5568a2 --- /dev/null +++ b/roles/prereq_network_dns/templates/networkmanager.conf.j2 @@ -0,0 +1,8 @@ +# Managed by Ansible +# See https://access.redhat.com/solutions/3078301 + +[main] +dns = default + +[global-dns-domain-*] +servers={{ network_dns_forwarders | join(',') }} diff --git a/roles/prereq_network_dns/templates/resolv.conf.j2 b/roles/prereq_network_dns/templates/resolv.conf.j2 new file mode 100644 index 00000000..7bac2687 --- /dev/null +++ b/roles/prereq_network_dns/templates/resolv.conf.j2 @@ -0,0 +1,3 @@ +# Generated by Ansible +search {{ network_dns_domain }} +{{ ['nameserver'] | product(network_dns_forwarders) | map('join', ' ') | join('\n') }} diff --git a/roles/prereq_network_dns/vars/default.yml b/roles/prereq_network_dns/vars/default.yml new file mode 100644 index 00000000..351eb094 --- /dev/null +++ b/roles/prereq_network_dns/vars/default.yml @@ -0,0 +1,13 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 8d5a1e684db09ae646f6b02eeda43277afd1033c Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:06 -0400 Subject: [PATCH 36/72] Add Apache NiFi prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_nifi/README.md | 53 +++ roles/prereq_nifi/meta/argument_specs.yml | 23 ++ .../prereq_nifi/molecule/default/converge.yml | 23 ++ roles/prereq_nifi/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_nifi/molecule/default/destroy.yml | 157 ++++++++ .../prereq_nifi/molecule/default/molecule.yml | 49 +++ .../prereq_nifi/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_nifi/molecule/default/verify.yml | 36 ++ roles/prereq_nifi/tasks/main.yml | 39 ++ roles/prereq_nifi/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_nifi/README.md create mode 100644 roles/prereq_nifi/meta/argument_specs.yml create mode 100644 roles/prereq_nifi/molecule/default/converge.yml create mode 100644 roles/prereq_nifi/molecule/default/create.yml create mode 100644 roles/prereq_nifi/molecule/default/destroy.yml create mode 100644 roles/prereq_nifi/molecule/default/molecule.yml create mode 100644 roles/prereq_nifi/molecule/default/prepare.yml create mode 100644 roles/prereq_nifi/molecule/default/requirements.yml create mode 100644 roles/prereq_nifi/molecule/default/verify.yml create mode 100644 roles/prereq_nifi/tasks/main.yml create mode 100644 roles/prereq_nifi/vars/main.yml diff --git a/roles/prereq_nifi/README.md b/roles/prereq_nifi/README.md new file mode 100644 index 00000000..5976db02 --- /dev/null +++ b/roles/prereq_nifi/README.md @@ -0,0 +1,53 @@ +# prereq_nifi + +Set up for NiFi + +This role prepares a host for Apache NiFi usage by creating a dedicated system user and group named `nifi`. This user is essential for running NiFi processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure NiFi communication. + +The role will: +- Create the `nifi` system user and group. +- Configure home directories and other necessary local paths for the `nifi` user, if required. +- Ensure appropriate permissions are set for files and directories related to NiFi. +- Configure TLS ACLs to secure NiFi communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: nifi_nodes + tasks: + - name: Set up the nifi user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_nifi +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_nifi/meta/argument_specs.yml b/roles/prereq_nifi/meta/argument_specs.yml new file mode 100644 index 00000000..25147249 --- /dev/null +++ b/roles/prereq_nifi/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Nifi + description: | + Set up for Apache Nifi usage, notably, create the local C(nifi) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_nifi/molecule/default/converge.yml b/roles/prereq_nifi/molecule/default/converge.yml new file mode 100644 index 00000000..8d7184a4 --- /dev/null +++ b/roles/prereq_nifi/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Nifi + ansible.builtin.import_role: + name: cloudera.exe.prereq_nifi diff --git a/roles/prereq_nifi/molecule/default/create.yml b/roles/prereq_nifi/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_nifi/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_nifi/molecule/default/destroy.yml b/roles/prereq_nifi/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_nifi/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_nifi/molecule/default/molecule.yml b/roles/prereq_nifi/molecule/default/molecule.yml new file mode 100644 index 00000000..422b0cb8 --- /dev/null +++ b/roles/prereq_nifi/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_nifi-rhel9-4 + Project: Molecule testing for prereq_nifi +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_nifi/molecule/default/prepare.yml b/roles/prereq_nifi/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_nifi/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_nifi/molecule/default/requirements.yml b/roles/prereq_nifi/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_nifi/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_nifi/molecule/default/verify.yml b/roles/prereq_nifi/molecule/default/verify.yml new file mode 100644 index 00000000..f87d81d4 --- /dev/null +++ b/roles/prereq_nifi/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for nifi users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ nifi_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ nifi_local_accounts }}" diff --git a/roles/prereq_nifi/tasks/main.yml b/roles/prereq_nifi/tasks/main.yml new file mode 100644 index 00000000..09597100 --- /dev/null +++ b/roles/prereq_nifi/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ nifi_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ nifi_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_nifi/vars/main.yml b/roles/prereq_nifi/vars/main.yml new file mode 100644 index 00000000..fe4f8ff4 --- /dev/null +++ b/roles/prereq_nifi/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nifi_local_accounts: + - user: nifi + home: /var/lib/nifi + comment: Nifi + keystore_acl: true From 8fc13ef3bd9eb3813604bde2040e8af869f1ad02 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:07 -0400 Subject: [PATCH 37/72] Add Apache NiFi Registry prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_nifiregistry/README.md | 53 +++ .../meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_nifiregistry/tasks/main.yml | 39 ++ roles/prereq_nifiregistry/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_nifiregistry/README.md create mode 100644 roles/prereq_nifiregistry/meta/argument_specs.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/converge.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/create.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/destroy.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/molecule.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/prepare.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/requirements.yml create mode 100644 roles/prereq_nifiregistry/molecule/default/verify.yml create mode 100644 roles/prereq_nifiregistry/tasks/main.yml create mode 100644 roles/prereq_nifiregistry/vars/main.yml diff --git a/roles/prereq_nifiregistry/README.md b/roles/prereq_nifiregistry/README.md new file mode 100644 index 00000000..d774aeb2 --- /dev/null +++ b/roles/prereq_nifiregistry/README.md @@ -0,0 +1,53 @@ +# prereq_nifiregistry + +Set up for NiFi Registry + +This role prepares a host for Apache NiFi Registry usage by creating a dedicated system user and group named `nifiregistry`. This user is essential for running NiFi Registry processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure NiFi Registry communication. + +The role will: +- Create the `nifiregistry` system user and group. +- Configure home directories and other necessary local paths for the `nifiregistry` user, if required. +- Ensure appropriate permissions are set for files and directories related to NiFi Registry. +- Configure TLS ACLs to secure NiFi Registry communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: nifiregistry_nodes + tasks: + - name: Set up the nifiregistry user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_nifiregistry +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_nifiregistry/meta/argument_specs.yml b/roles/prereq_nifiregistry/meta/argument_specs.yml new file mode 100644 index 00000000..83350834 --- /dev/null +++ b/roles/prereq_nifiregistry/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for NiFi Registry + description: | + Set up for Apache NiFi Registry usage, notably, create the local C(nifiregistry) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_nifiregistry/molecule/default/converge.yml b/roles/prereq_nifiregistry/molecule/default/converge.yml new file mode 100644 index 00000000..b2abc61f --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Nifi Registry + ansible.builtin.import_role: + name: cloudera.exe.prereq_nifiregistry diff --git a/roles/prereq_nifiregistry/molecule/default/create.yml b/roles/prereq_nifiregistry/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_nifiregistry/molecule/default/destroy.yml b/roles/prereq_nifiregistry/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_nifiregistry/molecule/default/molecule.yml b/roles/prereq_nifiregistry/molecule/default/molecule.yml new file mode 100644 index 00000000..ab04958a --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_nifiregistry-rhel9-4 + Project: Molecule testing for prereq_nifiregistry +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_nifiregistry/molecule/default/prepare.yml b/roles/prereq_nifiregistry/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_nifiregistry/molecule/default/requirements.yml b/roles/prereq_nifiregistry/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_nifiregistry/molecule/default/verify.yml b/roles/prereq_nifiregistry/molecule/default/verify.yml new file mode 100644 index 00000000..fbb525ec --- /dev/null +++ b/roles/prereq_nifiregistry/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for nifi registry users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ nifiregistry_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ nifiregistry_local_accounts }}" diff --git a/roles/prereq_nifiregistry/tasks/main.yml b/roles/prereq_nifiregistry/tasks/main.yml new file mode 100644 index 00000000..5bf0d53e --- /dev/null +++ b/roles/prereq_nifiregistry/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ nifiregistry_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ nifiregistry_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_nifiregistry/vars/main.yml b/roles/prereq_nifiregistry/vars/main.yml new file mode 100644 index 00000000..0a750962 --- /dev/null +++ b/roles/prereq_nifiregistry/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nifiregistry_local_accounts: + - user: nifiregistry + home: /var/lib/nifiregistry + comment: Nifi Registry + keystore_acl: true From cc019be6bdc564d6d4e7cf740a58c800721bced3 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:08 -0400 Subject: [PATCH 38/72] Add NTP prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_ntp/README.md | 54 +++ roles/prereq_ntp/defaults/main.yml | 16 + roles/prereq_ntp/handlers/main.yml | 20 ++ roles/prereq_ntp/meta/argument_specs.yml | 23 ++ .../prereq_ntp/molecule/default/converge.yml | 23 ++ roles/prereq_ntp/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_ntp/molecule/default/destroy.yml | 157 ++++++++ .../prereq_ntp/molecule/default/molecule.yml | 43 +++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_ntp/molecule/default/verify.yml | 36 ++ roles/prereq_ntp/tasks/main.yml | 36 ++ roles/prereq_ntp/tests/inventory | 15 + roles/prereq_ntp/tests/test.yml | 20 ++ roles/prereq_ntp/vars/main.yml | 16 + 14 files changed, 816 insertions(+) create mode 100644 roles/prereq_ntp/README.md create mode 100644 roles/prereq_ntp/defaults/main.yml create mode 100644 roles/prereq_ntp/handlers/main.yml create mode 100644 roles/prereq_ntp/meta/argument_specs.yml create mode 100644 roles/prereq_ntp/molecule/default/converge.yml create mode 100644 roles/prereq_ntp/molecule/default/create.yml create mode 100644 roles/prereq_ntp/molecule/default/destroy.yml create mode 100644 roles/prereq_ntp/molecule/default/molecule.yml create mode 100644 roles/prereq_ntp/molecule/default/requirements.yml create mode 100644 roles/prereq_ntp/molecule/default/verify.yml create mode 100644 roles/prereq_ntp/tasks/main.yml create mode 100644 roles/prereq_ntp/tests/inventory create mode 100644 roles/prereq_ntp/tests/test.yml create mode 100644 roles/prereq_ntp/vars/main.yml diff --git a/roles/prereq_ntp/README.md b/roles/prereq_ntp/README.md new file mode 100644 index 00000000..fcd7b937 --- /dev/null +++ b/roles/prereq_ntp/README.md @@ -0,0 +1,54 @@ +# prereq_ntp + +Manage NTP Services + +This role manages NTP (Network Time Protocol) services on a host, with a focus on using Chrony as the preferred time synchronization service. It intelligently handles different states of NTP service installation and operation to ensure that a single, reliable time service is active. + +The role will: +- Check for the presence of both the `chrony` and `ntp` services. +- If neither service is installed or running, it will install the `chrony` package. +- If both `chrony` and `ntp` services are found to be installed and running, it will stop and disable the `ntp` service to prevent conflicts and prioritize `chrony`. +- Ensure the selected time synchronization service (`chrony`) is running and enabled on system boot. + +# Requirements + +- Root or `sudo` privileges are required on the target host to install and manage system packages and services. +- Network access from the target host to configured NTP servers for time synchronization. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Manage NTP services to prioritize Chrony + ansible.builtin.import_role: + name: cloudera.exe.prereq_ntp +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ntp/defaults/main.yml b/roles/prereq_ntp/defaults/main.yml new file mode 100644 index 00000000..a1ed182d --- /dev/null +++ b/roles/prereq_ntp/defaults/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# defaults file for prereq_ntp diff --git a/roles/prereq_ntp/handlers/main.yml b/roles/prereq_ntp/handlers/main.yml new file mode 100644 index 00000000..601c2b15 --- /dev/null +++ b/roles/prereq_ntp/handlers/main.yml @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Enable and start chronyd service + ansible.builtin.service: + name: chronyd + state: started + enabled: true diff --git a/roles/prereq_ntp/meta/argument_specs.yml b/roles/prereq_ntp/meta/argument_specs.yml new file mode 100644 index 00000000..37711504 --- /dev/null +++ b/roles/prereq_ntp/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Manage NTP Services + description: | + This module installs the Chrony NTP package if neither Chrony nor NTP is currently installed or running on the system. + If both services are present and running, it will stop the NTP service to prioritize Chrony. + author: "Ronald Suplina " + options: {} diff --git a/roles/prereq_ntp/molecule/default/converge.yml b/roles/prereq_ntp/molecule/default/converge.yml new file mode 100644 index 00000000..945b25ab --- /dev/null +++ b/roles/prereq_ntp/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Manage NTP Services + ansible.builtin.import_role: + name: cloudera.exe.prereq_ntp diff --git a/roles/prereq_ntp/molecule/default/create.yml b/roles/prereq_ntp/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_ntp/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ntp/molecule/default/destroy.yml b/roles/prereq_ntp/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_ntp/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ntp/molecule/default/molecule.yml b/roles/prereq_ntp/molecule/default/molecule.yml new file mode 100644 index 00000000..207087cd --- /dev/null +++ b/roles/prereq_ntp/molecule/default/molecule.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + region: us-east-2 + tags: + Name: molecule-prereq_ntp-rhel9-4 + Project: Molecule testing for prereq_ntp +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_ntp/molecule/default/requirements.yml b/roles/prereq_ntp/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_ntp/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_ntp/molecule/default/verify.yml b/roles/prereq_ntp/molecule/default/verify.yml new file mode 100644 index 00000000..cf67c9ed --- /dev/null +++ b/roles/prereq_ntp/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Check if chronyd or ntpd is installed and running + ansible.builtin.set_fact: + chronyd_running: "{{ 'chronyd.service' in ansible_facts.services and ansible_facts.services['chronyd.service'].state == 'running' }}" + ntpd_running: "{{ 'ntpd.service' in ansible_facts.services and ansible_facts.services['ntpd.service'].state == 'running' }}" + + - name: Fail if neither Chrony nor NTP service is running + ansible.builtin.fail: + msg: "Neither Chrony nor NTP service is running." + when: not chronyd_running and not ntpd_running + + - name: Fail if both Chrony and NTP service is running + ansible.builtin.fail: + msg: "Neither Chrony nor NTP service is running." + when: chronyd_running and ntpd_running diff --git a/roles/prereq_ntp/tasks/main.yml b/roles/prereq_ntp/tasks/main.yml new file mode 100644 index 00000000..01d23f48 --- /dev/null +++ b/roles/prereq_ntp/tasks/main.yml @@ -0,0 +1,36 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Gather service facts + ansible.builtin.service_facts: + +- name: Check if chronyd or ntpd is installed and running + ansible.builtin.set_fact: + chronyd_running: "{{ 'chronyd.service' in ansible_facts.services and ansible_facts.services['chronyd.service'].state == 'running' }}" + ntpd_running: "{{ 'ntpd.service' in ansible_facts.services and ansible_facts.services['ntpd.service'].state == 'running' }}" + +- name: Stop ntpd service if both services are running + ansible.builtin.service: + name: ntpd + state: stopped + when: chronyd_running and ntpd_running + +- name: Install and enable Chrony if neither chronyd nor ntpd is running + ansible.builtin.package: + name: chrony + state: present + when: not chronyd_running and not ntpd_running + notify: + - Enable and start chronyd service diff --git a/roles/prereq_ntp/tests/inventory b/roles/prereq_ntp/tests/inventory new file mode 100644 index 00000000..6778d06c --- /dev/null +++ b/roles/prereq_ntp/tests/inventory @@ -0,0 +1,15 @@ +// Copyright 2024 Cloudera, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +localhost diff --git a/roles/prereq_ntp/tests/test.yml b/roles/prereq_ntp/tests/test.yml new file mode 100644 index 00000000..ab62706c --- /dev/null +++ b/roles/prereq_ntp/tests/test.yml @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Test + hosts: localhost + remote_user: root + roles: + - prereq_ntp diff --git a/roles/prereq_ntp/vars/main.yml b/roles/prereq_ntp/vars/main.yml new file mode 100644 index 00000000..6c73b995 --- /dev/null +++ b/roles/prereq_ntp/vars/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# vars file for prereq_ntp From df8946b8a06d0639f4951b7a01d24485d924f702 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:08 -0400 Subject: [PATCH 39/72] Add Apache Oozie prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_oozie/README.md | 53 +++ roles/prereq_oozie/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_oozie/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_oozie/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_oozie/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_oozie/molecule/default/verify.yml | 36 ++ roles/prereq_oozie/tasks/main.yml | 39 ++ roles/prereq_oozie/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_oozie/README.md create mode 100644 roles/prereq_oozie/meta/argument_specs.yml create mode 100644 roles/prereq_oozie/molecule/default/converge.yml create mode 100644 roles/prereq_oozie/molecule/default/create.yml create mode 100644 roles/prereq_oozie/molecule/default/destroy.yml create mode 100644 roles/prereq_oozie/molecule/default/molecule.yml create mode 100644 roles/prereq_oozie/molecule/default/prepare.yml create mode 100644 roles/prereq_oozie/molecule/default/requirements.yml create mode 100644 roles/prereq_oozie/molecule/default/verify.yml create mode 100644 roles/prereq_oozie/tasks/main.yml create mode 100644 roles/prereq_oozie/vars/main.yml diff --git a/roles/prereq_oozie/README.md b/roles/prereq_oozie/README.md new file mode 100644 index 00000000..6c380711 --- /dev/null +++ b/roles/prereq_oozie/README.md @@ -0,0 +1,53 @@ +# prereq_oozie + +Set up for Oozie + +This role prepares a host for Apache Oozie usage by creating a dedicated system user and group named `oozie`. This user is essential for running Oozie processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Oozie communication. + +The role will: +- Create the `oozie` system user and group. +- Configure home directories and other necessary local paths for the `oozie` user, if required. +- Ensure appropriate permissions are set for files and directories related to Oozie. +- Configure TLS ACLs to secure Oozie communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: oozie_nodes + tasks: + - name: Set up the oozie user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_oozie +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_oozie/meta/argument_specs.yml b/roles/prereq_oozie/meta/argument_specs.yml new file mode 100644 index 00000000..35f5cdb8 --- /dev/null +++ b/roles/prereq_oozie/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Oozie + description: | + Set up for Apache Oozie usage, notably, create the local C(oozie) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_oozie/molecule/default/converge.yml b/roles/prereq_oozie/molecule/default/converge.yml new file mode 100644 index 00000000..0b884646 --- /dev/null +++ b/roles/prereq_oozie/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Oozie + ansible.builtin.import_role: + name: cloudera.exe.prereq_oozie diff --git a/roles/prereq_oozie/molecule/default/create.yml b/roles/prereq_oozie/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_oozie/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_oozie/molecule/default/destroy.yml b/roles/prereq_oozie/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_oozie/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_oozie/molecule/default/molecule.yml b/roles/prereq_oozie/molecule/default/molecule.yml new file mode 100644 index 00000000..03b20180 --- /dev/null +++ b/roles/prereq_oozie/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_oozie-rhel9-4 + Project: Molecule testing for prereq_oozie +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_oozie/molecule/default/prepare.yml b/roles/prereq_oozie/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_oozie/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_oozie/molecule/default/requirements.yml b/roles/prereq_oozie/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_oozie/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_oozie/molecule/default/verify.yml b/roles/prereq_oozie/molecule/default/verify.yml new file mode 100644 index 00000000..0b19a7a9 --- /dev/null +++ b/roles/prereq_oozie/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for oozie users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ oozie_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ oozie_local_accounts }}" diff --git a/roles/prereq_oozie/tasks/main.yml b/roles/prereq_oozie/tasks/main.yml new file mode 100644 index 00000000..5141b50f --- /dev/null +++ b/roles/prereq_oozie/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ oozie_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ oozie_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_oozie/vars/main.yml b/roles/prereq_oozie/vars/main.yml new file mode 100644 index 00000000..10ae9d1d --- /dev/null +++ b/roles/prereq_oozie/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +oozie_local_accounts: + - user: oozie + home: /var/lib/oozie + comment: Oozie + keystore_acl: true From 3774c415dee0fbb3f2a25c7c97c7e67eee44b430 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:09 -0400 Subject: [PATCH 40/72] Add Apache Oozie database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_oozie_database/README.md | 79 ++++ roles/prereq_oozie_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_oozie_database/tasks/main.yml | 24 ++ 11 files changed, 853 insertions(+) create mode 100644 roles/prereq_oozie_database/README.md create mode 100644 roles/prereq_oozie_database/defaults/main.yml create mode 100644 roles/prereq_oozie_database/meta/argument_specs.yml create mode 100644 roles/prereq_oozie_database/molecule/default/converge.yml create mode 100644 roles/prereq_oozie_database/molecule/default/create.yml create mode 100644 roles/prereq_oozie_database/molecule/default/destroy.yml create mode 100644 roles/prereq_oozie_database/molecule/default/molecule.yml create mode 100644 roles/prereq_oozie_database/molecule/default/prepare.yml create mode 100644 roles/prereq_oozie_database/molecule/default/requirements.yml create mode 100644 roles/prereq_oozie_database/molecule/default/verify.yml create mode 100644 roles/prereq_oozie_database/tasks/main.yml diff --git a/roles/prereq_oozie_database/README.md b/roles/prereq_oozie_database/README.md new file mode 100644 index 00000000..45aec3c1 --- /dev/null +++ b/roles/prereq_oozie_database/README.md @@ -0,0 +1,79 @@ +# prereq_oozie_database + +Set up database and user accounts for Oozie + +This role automates the setup of a database and its associated user accounts specifically for Apache Oozie services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `oozie_database`. +- Create a new database user specified by `oozie_username` with the password from `oozie_password`. +- Grant ownership and all necessary privileges to the `oozie_username` for the new database. +- Ensure the database is configured correctly for Oozie operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `oozie_username` | `str` | `False` | `oozie` | The username for the Oozie database user. This user will also be the owner of the database. | +| `oozie_password` | `str` | `False` | `oozie` | The password for the Oozie database user. It is highly recommended to override this default in production. | +| `oozie_database` | `str` | `False` | `oozie` | The name of the database to be created for Oozie. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Oozie database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_oozie_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Oozie database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_oozie_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + oozie_username: "my_oozie_user" + oozie_password: "a_strong_oozie_password" + oozie_database: "my_oozie_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_oozie_database/defaults/main.yml b/roles/prereq_oozie_database/defaults/main.yml new file mode 100644 index 00000000..6dd8fc60 --- /dev/null +++ b/roles/prereq_oozie_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +oozie_username: oozie +oozie_password: oozie +oozie_database: oozie diff --git a/roles/prereq_oozie_database/meta/argument_specs.yml b/roles/prereq_oozie_database/meta/argument_specs.yml new file mode 100644 index 00000000..1b4b58a3 --- /dev/null +++ b/roles/prereq_oozie_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Oozie + description: + - Set up the Apache Oozie database and its associated user accounts, ensuring proper configuration for Oozie operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `oozie_username`, `oozie_password`, + and `oozie_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + oozie_username: + description: The username for the Oozie database user and owner of the database. + type: str + required: false + default: oozie + oozie_password: + description: The password for the Oozie database user. + type: str + required: false + default: oozie + oozie_database: + description: The name of the database to be created for Oozie. + type: str + required: false + default: oozie diff --git a/roles/prereq_oozie_database/molecule/default/converge.yml b/roles/prereq_oozie_database/molecule/default/converge.yml new file mode 100644 index 00000000..00d39622 --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Oozie database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_oozie_database diff --git a/roles/prereq_oozie_database/molecule/default/create.yml b/roles/prereq_oozie_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_oozie_database/molecule/default/destroy.yml b/roles/prereq_oozie_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_oozie_database/molecule/default/molecule.yml b/roles/prereq_oozie_database/molecule/default/molecule.yml new file mode 100644 index 00000000..e47bfd1f --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_oozie_database-rhel9-4 + Project: Molecule testing for prereq_oozie_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_oozie_database/molecule/default/prepare.yml b/roles/prereq_oozie_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_oozie_database/molecule/default/requirements.yml b/roles/prereq_oozie_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_oozie_database/molecule/default/verify.yml b/roles/prereq_oozie_database/molecule/default/verify.yml new file mode 100644 index 00000000..3185d90c --- /dev/null +++ b/roles/prereq_oozie_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'oozie';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database oozie does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'oozie';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User oozie does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_oozie_database/tasks/main.yml b/roles/prereq_oozie_database/tasks/main.yml new file mode 100644 index 00000000..df5f8bb8 --- /dev/null +++ b/roles/prereq_oozie_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ oozie_details }}" + oozie_details: + - user: "{{ oozie_username }}" + password: "{{ oozie_password }}" + db: "{{ oozie_database }}" + no_log: true From 6431f3607fb1fcc858503fa7c06fb3e92dd5b551 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:10 -0400 Subject: [PATCH 41/72] Add general OS prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_os/README.md | 58 +++ roles/prereq_os/defaults/main.yml | 16 + roles/prereq_os/meta/argument_specs.yml | 28 ++ roles/prereq_os/molecule/default/converge.yml | 23 ++ roles/prereq_os/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_os/molecule/default/destroy.yml | 157 ++++++++ roles/prereq_os/molecule/default/molecule.yml | 53 +++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_os/molecule/default/verify.yml | 40 +++ roles/prereq_os/tasks/main.yml | 38 ++ 10 files changed, 770 insertions(+) create mode 100644 roles/prereq_os/README.md create mode 100644 roles/prereq_os/defaults/main.yml create mode 100644 roles/prereq_os/meta/argument_specs.yml create mode 100644 roles/prereq_os/molecule/default/converge.yml create mode 100644 roles/prereq_os/molecule/default/create.yml create mode 100644 roles/prereq_os/molecule/default/destroy.yml create mode 100644 roles/prereq_os/molecule/default/molecule.yml create mode 100644 roles/prereq_os/molecule/default/requirements.yml create mode 100644 roles/prereq_os/molecule/default/verify.yml create mode 100644 roles/prereq_os/tasks/main.yml diff --git a/roles/prereq_os/README.md b/roles/prereq_os/README.md new file mode 100644 index 00000000..e8e4fef8 --- /dev/null +++ b/roles/prereq_os/README.md @@ -0,0 +1,58 @@ +# prereq_os + +Update general OS requirements. + +This role updates general operating system requirements to prepare a host for a Cloudera deployment. It sets the system's timezone and addresses specific file permission requirements for the global `/tmp` directory on certain operating systems. + +The role will: +- Set the host's timezone using the `os_timezone` variable. +- Ensure that the global `/tmp` directory has correct permissions to allow access for Cloudera Manager services. +- Specifically on Ubuntu 20.04, it will adjust root permissions on the global `/tmp` directory to comply with operational requirements. + +# Requirements + +- Root or `sudo` privileges are required on the target host to update the system timezone and modify directory permissions. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `os_timezone` | `str` | `False` | `UTC` | The timezone to set on the host. This should be a valid timezone string (e.g., `America/New_York`, `Europe/London`). | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Update OS requirements with default UTC timezone + ansible.builtin.import_role: + name: cloudera.exe.prereq_os + + - name: Update OS requirements with a specific timezone + ansible.builtin.import_role: + name: cloudera.exe.prereq_os + vars: + os_timezone: "America/Denver" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_os/defaults/main.yml b/roles/prereq_os/defaults/main.yml new file mode 100644 index 00000000..7225bf55 --- /dev/null +++ b/roles/prereq_os/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +os_timezone: "UTC" diff --git a/roles/prereq_os/meta/argument_specs.yml b/roles/prereq_os/meta/argument_specs.yml new file mode 100644 index 00000000..3407e2e6 --- /dev/null +++ b/roles/prereq_os/meta/argument_specs.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Update general OS requirements. + description: + - Update timezone. + - Update Cloudera Manager access to global C(/tmp) directory. + - Update Ubuntu 20.04 root permissions to global C(/tmp) directory. + author: + - Webster Mudge (wmudge@cloudera.com) + options: + os_timezone: + description: Timezone to set on the host. + default: "UTC" diff --git a/roles/prereq_os/molecule/default/converge.yml b/roles/prereq_os/molecule/default/converge.yml new file mode 100644 index 00000000..ecb1f9f3 --- /dev/null +++ b/roles/prereq_os/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Set OS-specific options + ansible.builtin.import_role: + name: cloudera.exe.prereq_os diff --git a/roles/prereq_os/molecule/default/create.yml b/roles/prereq_os/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_os/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_os/molecule/default/destroy.yml b/roles/prereq_os/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_os/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_os/molecule/default/molecule.yml b/roles/prereq_os/molecule/default/molecule.yml new file mode 100644 index 00000000..0981f050 --- /dev/null +++ b/roles/prereq_os/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_os-rhel9-4 + Project: Molecule testing for prereq_os + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_os-ubuntu20-04 + Project: Molecule testing for prereq_os +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_os/molecule/default/requirements.yml b/roles/prereq_os/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_os/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_os/molecule/default/verify.yml b/roles/prereq_os/molecule/default/verify.yml new file mode 100644 index 00000000..733e7d7d --- /dev/null +++ b/roles/prereq_os/molecule/default/verify.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: true + become: true + tasks: + - name: Discover /tmp details + ansible.builtin.stat: + path: /tmp + register: __tmp + failed_when: not __tmp.stat.exists + + - name: Check /tmp permissions + ansible.builtin.assert: + that: + - __tmp.stat.isdir + - __tmp.stat.mode == "1777" + + - name: Check root permissions for non-root files (Ubuntu 20.04) + when: + - ansible_distribution == 'Ubuntu' + - ansible_distribution_version == '20.04' + ansible.builtin.command: sysctl -n fs.protected_regular + register: __protected + failed_when: __protected.stdout != "0" + changed_when: false diff --git a/roles/prereq_os/tasks/main.yml b/roles/prereq_os/tasks/main.yml new file mode 100644 index 00000000..fd90f95b --- /dev/null +++ b/roles/prereq_os/tasks/main.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set timezone + community.general.timezone: + hwclock: "{{ os_timezone }}" + +- name: Configure /tmp directory + ansible.builtin.file: + path: /tmp + state: directory + mode: "1777" + +# See https://askubuntu.com/questions/1250974/user-root-cant-write-to-file-in-tmp-owned-by-someone-else-in-20-04-but-can-in +- name: Fix root permissions for non-root file (Ubuntu 20.04) + when: + - ansible_distribution == 'Ubuntu' + - ansible_distribution_version == '20.04' + block: + - name: Set 'protected_regular' kernel parameter + ansible.posix.sysctl: + sysctl_file: /usr/lib/sysctl.d/protect-links.conf + name: fs.protected_regular + value: 0 + state: present + reload: true From 00d65c87560fb82171cc9ab69b136c85177e9c0b Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:10 -0400 Subject: [PATCH 42/72] Add Apache Phoenix prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_phoenix/README.md | 52 +++ roles/prereq_phoenix/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 24 ++ roles/prereq_phoenix/tasks/main.yml | 39 ++ roles/prereq_phoenix/vars/main.yml | 22 ++ 10 files changed, 738 insertions(+) create mode 100644 roles/prereq_phoenix/README.md create mode 100644 roles/prereq_phoenix/meta/argument_specs.yml create mode 100644 roles/prereq_phoenix/molecule/default/converge.yml create mode 100644 roles/prereq_phoenix/molecule/default/create.yml create mode 100644 roles/prereq_phoenix/molecule/default/destroy.yml create mode 100644 roles/prereq_phoenix/molecule/default/molecule.yml create mode 100644 roles/prereq_phoenix/molecule/default/requirements.yml create mode 100644 roles/prereq_phoenix/molecule/default/verify.yml create mode 100644 roles/prereq_phoenix/tasks/main.yml create mode 100644 roles/prereq_phoenix/vars/main.yml diff --git a/roles/prereq_phoenix/README.md b/roles/prereq_phoenix/README.md new file mode 100644 index 00000000..ad7b54b8 --- /dev/null +++ b/roles/prereq_phoenix/README.md @@ -0,0 +1,52 @@ +# prereq_phoenix + +Set up for Phoenix + +This role prepares a host for Apache Phoenix usage by creating a dedicated system user and group named `phoenix`. This user is essential for running Phoenix processes with appropriate permissions and isolation. + +The role will: +- Create the `phoenix` system user and group. +- Configure home directories and other necessary local paths for the `phoenix` user, if required. +- Ensure appropriate permissions are set for files and directories related to Phoenix. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: phoenix_nodes + tasks: + - name: Set up the phoenix user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_phoenix +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_phoenix/meta/argument_specs.yml b/roles/prereq_phoenix/meta/argument_specs.yml new file mode 100644 index 00000000..881aaead --- /dev/null +++ b/roles/prereq_phoenix/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Phoenix + description: | + Set up for Apache Phoenix usage, notably, create local C(phoenix) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_phoenix/molecule/default/converge.yml b/roles/prereq_phoenix/molecule/default/converge.yml new file mode 100644 index 00000000..9302f16e --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Phoenix + ansible.builtin.import_role: + name: cloudera.exe.prereq_phoenix diff --git a/roles/prereq_phoenix/molecule/default/create.yml b/roles/prereq_phoenix/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_phoenix/molecule/default/destroy.yml b/roles/prereq_phoenix/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_phoenix/molecule/default/molecule.yml b/roles/prereq_phoenix/molecule/default/molecule.yml new file mode 100644 index 00000000..e78bd343 --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_phoenix-rhel9-4 + Project: Molecule testing for prereq_phoenix +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_phoenix/molecule/default/requirements.yml b/roles/prereq_phoenix/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_phoenix/molecule/default/verify.yml b/roles/prereq_phoenix/molecule/default/verify.yml new file mode 100644 index 00000000..0aa0f775 --- /dev/null +++ b/roles/prereq_phoenix/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check for phoenix user + ansible.builtin.command: grep phoenix /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false diff --git a/roles/prereq_phoenix/tasks/main.yml b/roles/prereq_phoenix/tasks/main.yml new file mode 100644 index 00000000..8a9c08c1 --- /dev/null +++ b/roles/prereq_phoenix/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.user: + name: "{{ account.user }}" + home: "{{ account.home }}" + comment: "{{ account.comment | default(omit) }}" + groups: "{{ account.extra_groups | default(omit) }}" + append: true + uid: "{{ account.uid | default(omit) }}" + shell: "{{ account.shell | default(phoenix_default_shell) }}" + loop: "{{ phoenix_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Set local user home directory permissions + ansible.builtin.file: + path: "{{ account.home }}" + owner: "{{ account.user }}" + group: "{{ account.user }}" + mode: "{{ account.mode | default(phoenix_default_home_dir_mode) }}" + loop: "{{ phoenix_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.home }}" diff --git a/roles/prereq_phoenix/vars/main.yml b/roles/prereq_phoenix/vars/main.yml new file mode 100644 index 00000000..e2c5d3a1 --- /dev/null +++ b/roles/prereq_phoenix/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +phoenix_local_accounts: + - user: phoenix + home: /var/lib/phoenix + comment: phoenix + +phoenix_default_shell: /sbin/nologin +phoenix_default_home_dir_mode: "0755" From 84e74c9022645e455ec3c607fcda689f6288c9c9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:11 -0400 Subject: [PATCH 43/72] Add psycopg2 prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_psycopg2/README.md | 60 ++++ roles/prereq_psycopg2/defaults/main.yml | 17 + roles/prereq_psycopg2/handlers/main.yml | 22 ++ roles/prereq_psycopg2/meta/argument_specs.yml | 28 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 53 +++ .../molecule/default/prepare.yml | 23 ++ .../molecule/default/requirements.yml | 20 ++ .../molecule/default/verify.yml | 24 ++ roles/prereq_psycopg2/tasks/Debian.yml | 19 + roles/prereq_psycopg2/tasks/RedHat.yml | 38 ++ roles/prereq_psycopg2/tasks/main.yml | 21 ++ 14 files changed, 841 insertions(+) create mode 100644 roles/prereq_psycopg2/README.md create mode 100644 roles/prereq_psycopg2/defaults/main.yml create mode 100644 roles/prereq_psycopg2/handlers/main.yml create mode 100644 roles/prereq_psycopg2/meta/argument_specs.yml create mode 100644 roles/prereq_psycopg2/molecule/default/converge.yml create mode 100644 roles/prereq_psycopg2/molecule/default/create.yml create mode 100644 roles/prereq_psycopg2/molecule/default/destroy.yml create mode 100644 roles/prereq_psycopg2/molecule/default/molecule.yml create mode 100644 roles/prereq_psycopg2/molecule/default/prepare.yml create mode 100644 roles/prereq_psycopg2/molecule/default/requirements.yml create mode 100644 roles/prereq_psycopg2/molecule/default/verify.yml create mode 100644 roles/prereq_psycopg2/tasks/Debian.yml create mode 100644 roles/prereq_psycopg2/tasks/RedHat.yml create mode 100644 roles/prereq_psycopg2/tasks/main.yml diff --git a/roles/prereq_psycopg2/README.md b/roles/prereq_psycopg2/README.md new file mode 100644 index 00000000..79990596 --- /dev/null +++ b/roles/prereq_psycopg2/README.md @@ -0,0 +1,60 @@ +# prereq_psycopg2 + +Install psycopg2 + +This role installs the `psycopg2` Python package, which is the most popular PostgreSQL database adapter for the Python programming language. It is a critical component for many applications that need to interact with a PostgreSQL database. The role also provides an option to manage the state of PostgreSQL repositories on the host during the installation process. + +The role will: +- Install the `psycopg2` Python package using `pip`. +- Optionally disable PostgreSQL-related package repositories before installation to prevent conflicts or unintended package updates, as controlled by `rdbms_repo_disable`. +- Ensure the package is installed and available for Python 3 applications. + +# Requirements + +- Python 3 and its package manager (`pip`) must be installed on the target host. +- Root or `sudo` privileges are required on the target host to install system-level packages and manage repositories, as `psycopg2` may have system library dependencies. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `rdbms_repo_disable` | `bool` | `False` | `true` | Flag to control whether PostgreSQL repositories are disabled before installing `psycopg2`. This is useful to prevent potential conflicts with existing repositories. If `false`, repositories will remain active. | + +# Example Playbook + +```yaml +- hosts: db_clients + tasks: + - name: Install psycopg2 and disable PSQL repos during installation + ansible.builtin.import_role: + name: cloudera.exe.prereq_psycopg2 + # The rdbms_repo_disable flag will be true by default. + + - name: Install psycopg2 without disabling PSQL repos + ansible.builtin.import_role: + name: cloudera.exe.prereq_psycopg2 + vars: + rdbms_repo_disable: false +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_psycopg2/defaults/main.yml b/roles/prereq_psycopg2/defaults/main.yml new file mode 100644 index 00000000..f2d15a86 --- /dev/null +++ b/roles/prereq_psycopg2/defaults/main.yml @@ -0,0 +1,17 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +rdbms_repo_disable: false diff --git a/roles/prereq_psycopg2/handlers/main.yml b/roles/prereq_psycopg2/handlers/main.yml new file mode 100644 index 00000000..f179a4ee --- /dev/null +++ b/roles/prereq_psycopg2/handlers/main.yml @@ -0,0 +1,22 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Clean YUM metadata + ansible.builtin.command: yum clean metadata + changed_when: false + +- name: Clean DNF metadata + ansible.builtin.command: dnf clean metadata + changed_when: false diff --git a/roles/prereq_psycopg2/meta/argument_specs.yml b/roles/prereq_psycopg2/meta/argument_specs.yml new file mode 100644 index 00000000..dcdfb524 --- /dev/null +++ b/roles/prereq_psycopg2/meta/argument_specs.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Install psycopg2" + description: + - Installs the psycopg2 Python package for PostgreSQL database. + author: + - "Jim Enright " + options: + rdbms_repo_disable: + description: Disable PSQL repositories before installing psycopg2 + type: bool + required: false + default: true diff --git a/roles/prereq_psycopg2/molecule/default/converge.yml b/roles/prereq_psycopg2/molecule/default/converge.yml new file mode 100644 index 00000000..9622c811 --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Install Psycopg2 + ansible.builtin.import_role: + name: cloudera.exe.prereq_psycopg2 diff --git a/roles/prereq_psycopg2/molecule/default/create.yml b/roles/prereq_psycopg2/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_psycopg2/molecule/default/destroy.yml b/roles/prereq_psycopg2/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_psycopg2/molecule/default/molecule.yml b/roles/prereq_psycopg2/molecule/default/molecule.yml new file mode 100644 index 00000000..4f3a67c4 --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_psycopg2-rhel9-4 + Project: Molecule testing for prereq_psycopg2 + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_psycopg2-ubuntu20-04 + Project: Molecule testing for prereq_psycopg2 +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_psycopg2/molecule/default/prepare.yml b/roles/prereq_psycopg2/molecule/default/prepare.yml new file mode 100644 index 00000000..8ca91e07 --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/prepare.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: true + become: true + tasks: + - name: Install Python + ansible.builtin.import_role: + name: cloudera.exe.prereq_python diff --git a/roles/prereq_psycopg2/molecule/default/requirements.yml b/roles/prereq_psycopg2/molecule/default/requirements.yml new file mode 100644 index 00000000..724080bb --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/requirements.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.utils + - community.general diff --git a/roles/prereq_psycopg2/molecule/default/verify.yml b/roles/prereq_psycopg2/molecule/default/verify.yml new file mode 100644 index 00000000..2f80472e --- /dev/null +++ b/roles/prereq_psycopg2/molecule/default/verify.yml @@ -0,0 +1,24 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + # Check 1 - confirm psycopg2 package exists + - name: Retrieve list of installed Python packages + community.general.pip_package_info: + register: __pip_packages + failed_when: __pip_packages.packages.pip.keys() is not search('psycopg2') diff --git a/roles/prereq_psycopg2/tasks/Debian.yml b/roles/prereq_psycopg2/tasks/Debian.yml new file mode 100644 index 00000000..5947b0a8 --- /dev/null +++ b/roles/prereq_psycopg2/tasks/Debian.yml @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install psycopg2 library + ansible.builtin.pip: + name: psycopg2-binary + state: present diff --git a/roles/prereq_psycopg2/tasks/RedHat.yml b/roles/prereq_psycopg2/tasks/RedHat.yml new file mode 100644 index 00000000..e6cbb45d --- /dev/null +++ b/roles/prereq_psycopg2/tasks/RedHat.yml @@ -0,0 +1,38 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Remove repositories and clean metadata + ansible.builtin.yum_repository: + name: "{{ repo }}" + state: absent + loop: + - pgdg-common + - pgdg + loop_control: + loop_var: repo + when: rdbms_repo_disable + notify: Clean YUM metadata + +- name: Install psycopg2 build prerequisites + ansible.builtin.package: + name: + - gcc + - libpq-devel + +# This fails if the PSQL repositories are not removed +- name: Install psycopg2 library + ansible.builtin.pip: + name: psycopg2-binary + state: present diff --git a/roles/prereq_psycopg2/tasks/main.yml b/roles/prereq_psycopg2/tasks/main.yml new file mode 100644 index 00000000..7043120b --- /dev/null +++ b/roles/prereq_psycopg2/tasks/main.yml @@ -0,0 +1,21 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Include OS-specific tasks to install psycopg2 + ansible.builtin.include_tasks: + file: "{{ item }}" + with_first_found: + - "{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_os_family }}.yml" From cbf5ff6bf166d2ecf14b04ca1f7744bdb87242ca Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:12 -0400 Subject: [PATCH 44/72] Add Python prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_python/README.md | 78 ++++ roles/prereq_python/defaults/main.yml | 22 ++ roles/prereq_python/meta/argument_specs.yml | 50 +++ .../molecule/default/converge.yml | 23 ++ .../prereq_python/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 53 +++ .../molecule/default/requirements.yml | 20 ++ .../prereq_python/molecule/default/verify.yml | 19 + roles/prereq_python/tasks/RedHat.yml | 55 +++ roles/prereq_python/tasks/Ubuntu.yml | 117 ++++++ roles/prereq_python/tasks/main.yml | 89 +++++ 12 files changed, 1019 insertions(+) create mode 100644 roles/prereq_python/README.md create mode 100644 roles/prereq_python/defaults/main.yml create mode 100644 roles/prereq_python/meta/argument_specs.yml create mode 100644 roles/prereq_python/molecule/default/converge.yml create mode 100644 roles/prereq_python/molecule/default/create.yml create mode 100644 roles/prereq_python/molecule/default/destroy.yml create mode 100644 roles/prereq_python/molecule/default/molecule.yml create mode 100644 roles/prereq_python/molecule/default/requirements.yml create mode 100644 roles/prereq_python/molecule/default/verify.yml create mode 100644 roles/prereq_python/tasks/RedHat.yml create mode 100644 roles/prereq_python/tasks/Ubuntu.yml create mode 100644 roles/prereq_python/tasks/main.yml diff --git a/roles/prereq_python/README.md b/roles/prereq_python/README.md new file mode 100644 index 00000000..6a21d770 --- /dev/null +++ b/roles/prereq_python/README.md @@ -0,0 +1,78 @@ +# prereq_python + +Install Python + +This role ensures that the correct versions of Python are installed on a host to meet the requirements of specified Cloudera Manager and Cloudera Runtime versions. It also handles the installation and update of the `pip` package manager for Python 3. For environments that still require Python 2, the role provides a flag to control whether the installation is done via the system package manager or from source. + +To validate the required Python version for a given Cloudera Manager and Runtime, the [Cloudera on premise documentation](https://docs.cloudera.com/cdp-private-cloud-base/latest/installation/topics/cdpdc-cm-install-python-3.8.html) and the support matrix variables defined in the `cloudera.exe.prereq_supported` role are used. + +The role will: +- Determine the required Python versions based on `cloudera_manager_version` and `cloudera_runtime_version`. +- Install the necessary Python 3 packages if a supported version is not already present. +- Install or update the `pip` package for Python 3. +- If Python 2 is required, install it either via the system's package manager (`python2_package_install: true`) or from source (`python2_package_install: false`). +- Ensure that the installed Python versions and `pip` are properly configured and accessible. + +# Requirements + +- Root or `sudo` privileges are required on the target host to install system packages. +- Network access to package repositories (for system packages) and PyPI (for pip). +- If installing Python 2 from source, the host will require build tools (e.g., `gcc`, `make`, development headers). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cloudera_manager_version` | `str` | `False` | `7.11.3` | The version of Cloudera Manager to use for determining Python version requirements. | +| `cloudera_runtime_version` | `str` | `False` | `7.1.9` | The version of Cloudera Runtime to use for determining Python version requirements. | +| `python3_package` | `str` | `False` | - | An optional name of the Python 3 package to install. This is only used when a supported version of Python 3 is not found. If not specified, the role will use OS-specific default package names. | +| `python3_pip_package` | `str` | `False` | `python-pip` | The name of the Python 3 Pip package to be installed or updated. | +| `python2_package_install` | `bool` | `False` | `true` | Flag to specify if Python 2 should be installed via the system package manager. If `false`, the role will attempt to install Python 2 from source. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Install Python for Cloudera Manager 7.11.3 / Cloudera Runtime 7.1.9 + ansible.builtin.import_role: + name: cloudera.exe.prereq_python + # All variables will use their defaults, installing Python 3 with pip and Python 2 via package manager. + + - name: Install Python for a different version of Cloudera Runtime + ansible.builtin.import_role: + name: cloudera.exe.prereq_python + vars: + cloudera_runtime_version: "7.1.8" + # The role will adjust Python version requirements accordingly. + + - name: Install Python with custom package names and install Python 2 from source + ansible.builtin.import_role: + name: cloudera.exe.prereq_python + vars: + python3_package: "python39" # Example custom package name for Python 3.9 + python3_pip_package: "python3-pip" + python2_package_install: false # Install Python 2 from source +``` + +# License + +``` +Copyright 2025 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` \ No newline at end of file diff --git a/roles/prereq_python/defaults/main.yml b/roles/prereq_python/defaults/main.yml new file mode 100644 index 00000000..fa16717f --- /dev/null +++ b/roles/prereq_python/defaults/main.yml @@ -0,0 +1,22 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +cloudera_manager_version: 7.11.3 +cloudera_runtime_version: 7.1.9 + +# python3_package: +python3_pip_package: python-pip + +python2_package_install: true diff --git a/roles/prereq_python/meta/argument_specs.yml b/roles/prereq_python/meta/argument_specs.yml new file mode 100644 index 00000000..ac638b51 --- /dev/null +++ b/roles/prereq_python/meta/argument_specs.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Install Python" + description: + - Ensures that the supported versions of Python for specified versions of Cloudera Manager and Runtime. + - Also ensures that Pip is installed and updated. + author: + - "Jim Enright " + options: + cloudera_manager_version: + description: Version of Cloudera Manager + type: "str" + default: "7.11.3" + cloudera_runtime_version: + description: Version of Cloudera Runtime + type: "str" + default: "7.1.9" + python3_package: + description: + - Optional Python3 package to be installed. + - This is used only when a supported version of Python3 is not already installed. + type: str + python3_pip_package: + description: + - Python3 Pip package to be installed. + type: str + default: python-pip + python2_package_install: + description: + - Flag to specify if Python2, should be installed via system package manager. + - This is used only when Python2 is required and not already installed. + - If I(True) Python2 will be installed using pacakage manager. + - If I(False) Python2 will be installed from source. + type: bool + default: true diff --git a/roles/prereq_python/molecule/default/converge.yml b/roles/prereq_python/molecule/default/converge.yml new file mode 100644 index 00000000..45a35510 --- /dev/null +++ b/roles/prereq_python/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Run OS python prereqs role + ansible.builtin.import_role: + name: cloudera.exe.prereq_python diff --git a/roles/prereq_python/molecule/default/create.yml b/roles/prereq_python/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_python/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_python/molecule/default/destroy.yml b/roles/prereq_python/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_python/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_python/molecule/default/molecule.yml b/roles/prereq_python/molecule/default/molecule.yml new file mode 100644 index 00000000..be876da0 --- /dev/null +++ b/roles/prereq_python/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_python-rhel9-4 + Project: Molecule testing for prereq_python + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_python-ubuntu20-04 + Project: Molecule testing for prereq_python +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_python/molecule/default/requirements.yml b/roles/prereq_python/molecule/default/requirements.yml new file mode 100644 index 00000000..724080bb --- /dev/null +++ b/roles/prereq_python/molecule/default/requirements.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.utils + - community.general diff --git a/roles/prereq_python/molecule/default/verify.yml b/roles/prereq_python/molecule/default/verify.yml new file mode 100644 index 00000000..aedb5736 --- /dev/null +++ b/roles/prereq_python/molecule/default/verify.yml @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify + hosts: all + gather_facts: false + # TODO: Set up tests for prereq_python role diff --git a/roles/prereq_python/tasks/RedHat.yml b/roles/prereq_python/tasks/RedHat.yml new file mode 100644 index 00000000..8a0fe59d --- /dev/null +++ b/roles/prereq_python/tasks/RedHat.yml @@ -0,0 +1,55 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Python3 and Pip +- name: Install supported version of Python3 if required + when: + - not __python3_version is version(__required_python_version, 'ge', version_type='pep440') + block: + - name: Install python3 package if defined + when: python3_package is defined + ansible.builtin.package: + name: "{{ python3_package }}" + state: present + + - name: Download and install + when: python3_package is not defined + ansible.builtin.debug: + msg: + - "TODO - install Python version {{ __required_python_version }}" + +- name: Ensure pip is installed + ansible.builtin.package: + name: "{{ python3_pip_package }}" + state: present + +- name: Ensure pip is upgraded + ansible.builtin.pip: + name: pip + extra_args: --upgrade + +# Python2 and Pip +- name: Ensure supported version of Python2 + when: + - __required_python2_version is defined + block: + - name: Install Python2 if required + when: + - not __python2_version is version(__required_python2_version, 'ge', version_type='pep440') + block: + - name: Download and install + ansible.builtin.debug: + msg: + - "TODO - install Python version {{ __required_python2_version }}" diff --git a/roles/prereq_python/tasks/Ubuntu.yml b/roles/prereq_python/tasks/Ubuntu.yml new file mode 100644 index 00000000..744d889c --- /dev/null +++ b/roles/prereq_python/tasks/Ubuntu.yml @@ -0,0 +1,117 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Python3 and Pip +- name: Install supported version of Python3 if required + when: + - not __python3_version is version(__required_python_version, 'ge', version_type='pep440') + block: + - name: Ensure required dependencies are installed + ansible.builtin.package: + name: + - build-essential + - zlib1g-dev + - libncurses5-dev + - libgdbm-dev + - libnss3-dev + - libssl-dev + - libreadline-dev + - libffi-dev + - curl + - libsqlite3-dev + - libbz2-dev + state: present + update_cache: true + + - name: Extract Python3 source + ansible.builtin.unarchive: + src: https://www.python.org/ftp/python/{{ __required_python_version }}/Python-{{ __required_python_version }}.tgz + dest: /tmp + remote_src: true + + - name: Configure Python3 for installation + ansible.builtin.command: ./configure --enable-optimizations + args: + chdir: /tmp/Python-{{ __required_python_version }} + changed_when: true + + - name: Build Python3 'all' target with extra arguments + community.general.make: + chdir: /tmp/Python-{{ __required_python_version }} + target: altinstall + +- name: Ensure pip is installed + ansible.builtin.package: + name: python3-pip + state: present + +- name: Ensure pip is upgraded + ansible.builtin.pip: + name: pip + extra_args: --upgrade + +# Python2 and Pip +- name: Ensure supported version of Python2 + when: + - __required_python2_version is defined + block: + - name: Install Python2 if required + when: + - not (__python2_version is version(__required_python2_version, 'ge', version_type='pep440')) + block: + # Install Python2 via system package + - name: Install Python2 package + when: python2_package_install + ansible.builtin.package: + name: "python{{ __required_python2_version }}" + state: present + + # Install Python2 via source + - name: Install Python2 source + when: not python2_package_install + block: + - name: Ensure required dependencies are installed + ansible.builtin.package: + name: + - build-essential + - zlib1g-dev + - libncurses5-dev + - libgdbm-dev + - libnss3-dev + - libssl-dev + - libreadline-dev + - libffi-dev + - curl + - libsqlite3-dev + - libbz2-dev + state: present + update_cache: true + + - name: Extract Python2 source + ansible.builtin.unarchive: + src: https://www.python.org/ftp/python/{{ __required_python2_version }}/Python-{{ __required_python2_version }}.tgz + dest: /tmp + remote_src: true + + - name: Configure Python2 for installation + ansible.builtin.command: ./configure --enable-optimizations + args: + chdir: /tmp/Python-{{ __required_python2_version }} + changed_when: true + + - name: Build Python2 'all' target with extra arguments + community.general.make: + chdir: /tmp/Python-{{ __required_python2_version }} + target: altinstall diff --git a/roles/prereq_python/tasks/main.yml b/roles/prereq_python/tasks/main.yml new file mode 100644 index 00000000..6c9b460c --- /dev/null +++ b/roles/prereq_python/tasks/main.yml @@ -0,0 +1,89 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Load support matrix variables for OS + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['distribution'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/{{ ansible_facts['os_family'] }}.yml" + - "{{ role_path }}/../prereq_supported/vars/default.yml" + +- name: Set required python version for specified runtime and manager versions + ansible.builtin.set_fact: + __required_python_version: >- + {{ + support_matrix | + selectattr('manager_version', '==', (cm_version.major + '.' + cm_version.minor + '.' + cm_version.patch)) | + selectattr('runtime_version', '==', ( + cloudera_runtime_version | regex_search('(\\d+\\.\\d+\\.\\d+)') + )) | + map(attribute='python_version') | + first + }} + __required_python2_version: >- + {{ + ( + support_matrix | + selectattr('manager_version', '==', (cm_version.major + '.' + cm_version.minor + '.' + cm_version.patch)) | + selectattr('runtime_version', '==', ( + cloudera_runtime_version | regex_search('(\\d+\\.\\d+\\.\\d+)') + )) | + map(attribute='python2') | + first + ) | + default(omit) + }} + __python3_version: "{{ ansible_python_version }}" + vars: + cm_version: "{{ cloudera_manager_version | cloudera.exe.cm_version }}" + +- name: If required check if Python2 is installed + when: + - __required_python2_version is defined + block: + - name: Check python2 version + ansible.builtin.command: python2 --version + register: __py2_check + changed_when: false + ignore_errors: true + + - name: Set facts for Python versions + ansible.builtin.set_fact: + __python2_installed: "{{ __py2_check.rc == 0 }}" + __python2_version: "{{ (__py2_check.stdout.split(' ')[1]) if __py2_check.rc == 0 else '0.0.0' }}" + +- name: Print Python versions + ansible.builtin.debug: + msg: + - "Required Python version: {{ __required_python_version }}" + - "Python version installed: {{ __python3_version }}" + - "Python2 required: {{ True if __required_python2_version is defined else False }}" + - "Python2 vesion installed: {{ __python2_version if __python2_version is defined else 'N/A' }}" + verbosity: 2 + +- name: Include OS-specific tasks to update or install Python + ansible.builtin.include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" From 6a72e10e617372e6437a944af492fba8a3e7a2f2 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:12 -0400 Subject: [PATCH 45/72] Add Hue Query Processor database prerequisites role Signed-off-by: Webster Mudge --- .../prereq_query_processor_database/README.md | 79 ++++ .../defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ .../tasks/main.yml | 24 ++ 11 files changed, 853 insertions(+) create mode 100644 roles/prereq_query_processor_database/README.md create mode 100644 roles/prereq_query_processor_database/defaults/main.yml create mode 100644 roles/prereq_query_processor_database/meta/argument_specs.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/converge.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/create.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/destroy.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/molecule.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/prepare.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/requirements.yml create mode 100644 roles/prereq_query_processor_database/molecule/default/verify.yml create mode 100644 roles/prereq_query_processor_database/tasks/main.yml diff --git a/roles/prereq_query_processor_database/README.md b/roles/prereq_query_processor_database/README.md new file mode 100644 index 00000000..1d9e4595 --- /dev/null +++ b/roles/prereq_query_processor_database/README.md @@ -0,0 +1,79 @@ +# prereq_query_processor_database + +Set up database and user accounts for Query Processor + +This role automates the setup of a database and its associated user accounts specifically for Hue Query Processor services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `query_processor_database`. +- Create a new database user specified by `query_processor_username` with the password from `query_processor_password`. +- Grant ownership and all necessary privileges to the `query_processor_username` for the new database. +- Ensure the database is configured correctly for Query Processor operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `query_processor_username` | `str` | `False` | `queryprocessor` | The username for the Query Processor database user. This user will also be the owner of the database. | +| `query_processor_password` | `str` | `False` | `queryprocessor` | The password for the Query Processor database user. It is highly recommended to override this default in production. | +| `query_processor_database` | `str` | `False` | `queryprocessor` | The name of the database to be created for Query Processor. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Query Processor database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_query_processor_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Query Processor database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_query_processor_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + query_processor_username: "my_qp_user" + query_processor_password: "a_strong_qp_password" + query_processor_database: "my_qp_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_query_processor_database/defaults/main.yml b/roles/prereq_query_processor_database/defaults/main.yml new file mode 100644 index 00000000..67edb895 --- /dev/null +++ b/roles/prereq_query_processor_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +query_processor_username: queryprocessor +query_processor_password: queryprocessor +query_processor_database: queryprocessor diff --git a/roles/prereq_query_processor_database/meta/argument_specs.yml b/roles/prereq_query_processor_database/meta/argument_specs.yml new file mode 100644 index 00000000..6dc7d1e0 --- /dev/null +++ b/roles/prereq_query_processor_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Query Processor + description: + - Set up the Hue Query Processor database and its associated user accounts, ensuring proper configuration for Query Processor operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `query_processor_username`, + `query_processor_password`, and `query_processor_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + query_processor_username: + description: The username for the Query Processor database user and owner of the database. + type: str + required: false + default: queryprocessor + query_processor_password: + description: The password for the Query Processor database user. + type: str + required: false + default: queryprocessor + query_processor_database: + description: The name of the database to be created for Query Processor. + type: str + required: false + default: queryprocessor diff --git a/roles/prereq_query_processor_database/molecule/default/converge.yml b/roles/prereq_query_processor_database/molecule/default/converge.yml new file mode 100644 index 00000000..c29ab0ed --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Query Processor database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_query_processor_database diff --git a/roles/prereq_query_processor_database/molecule/default/create.yml b/roles/prereq_query_processor_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_query_processor_database/molecule/default/destroy.yml b/roles/prereq_query_processor_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_query_processor_database/molecule/default/molecule.yml b/roles/prereq_query_processor_database/molecule/default/molecule.yml new file mode 100644 index 00000000..0a38f7b7 --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_query_processor_database-rhel9-4 + Project: Molecule testing for prereq_query_processor_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_query_processor_database/molecule/default/prepare.yml b/roles/prereq_query_processor_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_query_processor_database/molecule/default/requirements.yml b/roles/prereq_query_processor_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_query_processor_database/molecule/default/verify.yml b/roles/prereq_query_processor_database/molecule/default/verify.yml new file mode 100644 index 00000000..8fc943e4 --- /dev/null +++ b/roles/prereq_query_processor_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'queryprocessor';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database queryprocessor does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'queryprocessor';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User queryprocessor does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_query_processor_database/tasks/main.yml b/roles/prereq_query_processor_database/tasks/main.yml new file mode 100644 index 00000000..1e3e079d --- /dev/null +++ b/roles/prereq_query_processor_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ query_processor_details }}" + query_processor_details: + - user: "{{ query_processor_username }}" + password: "{{ query_processor_password }}" + db: "{{ query_processor_database }}" + no_log: true From a00963f37b90db57ab11e7430e4d2ba1cea1c6df Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:13 -0400 Subject: [PATCH 46/72] Add Apache Ranger prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_ranger/README.md | 54 +++ roles/prereq_ranger/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_ranger/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_ranger/molecule/default/verify.yml | 42 +++ roles/prereq_ranger/tasks/main.yml | 44 +++ roles/prereq_ranger/vars/main.yml | 28 ++ 11 files changed, 815 insertions(+) create mode 100644 roles/prereq_ranger/README.md create mode 100644 roles/prereq_ranger/meta/argument_specs.yml create mode 100644 roles/prereq_ranger/molecule/default/converge.yml create mode 100644 roles/prereq_ranger/molecule/default/create.yml create mode 100644 roles/prereq_ranger/molecule/default/destroy.yml create mode 100644 roles/prereq_ranger/molecule/default/molecule.yml create mode 100644 roles/prereq_ranger/molecule/default/prepare.yml create mode 100644 roles/prereq_ranger/molecule/default/requirements.yml create mode 100644 roles/prereq_ranger/molecule/default/verify.yml create mode 100644 roles/prereq_ranger/tasks/main.yml create mode 100644 roles/prereq_ranger/vars/main.yml diff --git a/roles/prereq_ranger/README.md b/roles/prereq_ranger/README.md new file mode 100644 index 00000000..916c6cd8 --- /dev/null +++ b/roles/prereq_ranger/README.md @@ -0,0 +1,54 @@ +# prereq_ranger + +Set up for Ranger + +This role prepares a host for Apache Ranger usage by creating a set of dedicated system users and a group. It specifically creates the `ranger`, `rangerraz`, and `rangertagsync` users and the `ranger` group. These users and groups are essential for running Ranger services with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Ranger communication. + +The role will: +- Create the `ranger` system group. +- Create the `ranger`, `rangerraz`, and `rangertagsync` system users, assigning them to the `ranger` group. +- Configure home directories and other necessary local paths for these users, if required. +- Ensure appropriate permissions are set for files and directories related to Ranger. +- Configure TLS ACLs to secure Ranger communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: ranger_nodes + tasks: + - name: Set up the ranger users and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_ranger +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ranger/meta/argument_specs.yml b/roles/prereq_ranger/meta/argument_specs.yml new file mode 100644 index 00000000..3df01715 --- /dev/null +++ b/roles/prereq_ranger/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Ranger + description: | + Set up for Apache Ranger usage, notably, create the local C(ranger,rangerraz,rangertagsync) users and local C(ranger) group. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_ranger/molecule/default/converge.yml b/roles/prereq_ranger/molecule/default/converge.yml new file mode 100644 index 00000000..86343a30 --- /dev/null +++ b/roles/prereq_ranger/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Ranger + ansible.builtin.import_role: + name: cloudera.exe.prereq_ranger diff --git a/roles/prereq_ranger/molecule/default/create.yml b/roles/prereq_ranger/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_ranger/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ranger/molecule/default/destroy.yml b/roles/prereq_ranger/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_ranger/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ranger/molecule/default/molecule.yml b/roles/prereq_ranger/molecule/default/molecule.yml new file mode 100644 index 00000000..7078a7d0 --- /dev/null +++ b/roles/prereq_ranger/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_ranger-rhel9-4 + Project: Molecule testing for prereq_ranger +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_ranger/molecule/default/prepare.yml b/roles/prereq_ranger/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_ranger/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_ranger/molecule/default/requirements.yml b/roles/prereq_ranger/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_ranger/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_ranger/molecule/default/verify.yml b/roles/prereq_ranger/molecule/default/verify.yml new file mode 100644 index 00000000..37d00043 --- /dev/null +++ b/roles/prereq_ranger/molecule/default/verify.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for ranger-related users + ansible.builtin.command: grep {{ item }} /etc/passwd + loop: + - ranger + - rangerraz + - rangertagsync + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for ranger group membership + ansible.builtin.command: groups ranger + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("ranger") + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ ranger_local_accounts }}" diff --git a/roles/prereq_ranger/tasks/main.yml b/roles/prereq_ranger/tasks/main.yml new file mode 100644 index 00000000..ff24db00 --- /dev/null +++ b/roles/prereq_ranger/tasks/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create ranger group + ansible.builtin.group: + name: ranger + state: present + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ ranger_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ ranger_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_ranger/vars/main.yml b/roles/prereq_ranger/vars/main.yml new file mode 100644 index 00000000..c706edb9 --- /dev/null +++ b/roles/prereq_ranger/vars/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ranger_local_accounts: + - user: ranger + home: /var/lib/ranger + comment: Ranger + extra_groups: [hadoop] + - user: rangerraz + home: /var/lib/rangerraz + comment: Ranger Raz User + extra_groups: [ranger, hadoop] + - user: rangertagsync + home: /var/lib/rangertagsync + comment: Ranger Tagsync User + extra_groups: [ranger, hadoop] From 5342d15fa779cf4de66ed505d87116111b2235af Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:14 -0400 Subject: [PATCH 47/72] Add Apache Ranger database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_ranger_database/README.md | 79 ++++ .../prereq_ranger_database/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_ranger_database/tasks/main.yml | 24 ++ 11 files changed, 853 insertions(+) create mode 100644 roles/prereq_ranger_database/README.md create mode 100644 roles/prereq_ranger_database/defaults/main.yml create mode 100644 roles/prereq_ranger_database/meta/argument_specs.yml create mode 100644 roles/prereq_ranger_database/molecule/default/converge.yml create mode 100644 roles/prereq_ranger_database/molecule/default/create.yml create mode 100644 roles/prereq_ranger_database/molecule/default/destroy.yml create mode 100644 roles/prereq_ranger_database/molecule/default/molecule.yml create mode 100644 roles/prereq_ranger_database/molecule/default/prepare.yml create mode 100644 roles/prereq_ranger_database/molecule/default/requirements.yml create mode 100644 roles/prereq_ranger_database/molecule/default/verify.yml create mode 100644 roles/prereq_ranger_database/tasks/main.yml diff --git a/roles/prereq_ranger_database/README.md b/roles/prereq_ranger_database/README.md new file mode 100644 index 00000000..a4417d96 --- /dev/null +++ b/roles/prereq_ranger_database/README.md @@ -0,0 +1,79 @@ +# prereq_ranger_database + +Set up database and user accounts for Ranger + +This role automates the setup of a database and its associated user accounts specifically for Apache Ranger services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `ranger_database`. +- Create a new database user specified by `ranger_username` with the password from `ranger_password`. +- Grant ownership and all necessary privileges to the `ranger_username` for the new database. +- Ensure the database is configured correctly for Ranger operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `ranger_username` | `str` | `False` | `rangeradmin` | The username for the Ranger database user. This user will also be the owner of the database. | +| `ranger_password` | `str` | `False` | `ranger` | The password for the Ranger database user. It is highly recommended to override this default in production. | +| `ranger_database` | `str` | `False` | `ranger` | The name of the database to be created for Ranger. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Ranger database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_ranger_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Ranger database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_ranger_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + ranger_username: "my_ranger_user" + ranger_password: "a_strong_ranger_password" + ranger_database: "my_ranger_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ranger_database/defaults/main.yml b/roles/prereq_ranger_database/defaults/main.yml new file mode 100644 index 00000000..ac5ac4c7 --- /dev/null +++ b/roles/prereq_ranger_database/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +ranger_username: rangeradmin +ranger_password: ranger +ranger_database: ranger diff --git a/roles/prereq_ranger_database/meta/argument_specs.yml b/roles/prereq_ranger_database/meta/argument_specs.yml new file mode 100644 index 00000000..f079e1b1 --- /dev/null +++ b/roles/prereq_ranger_database/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Ranger + description: + - Set up the Apache Ranger database and its associated user accounts, ensuring proper configuration for Ranger operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(ranger_username), + I(ranger_password), and I(ranger_database) variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + ranger_username: + description: The username for the Ranger database user and owner of the database. + type: str + required: false + default: rangeradmin + ranger_password: + description: The password for the Ranger database user. + type: str + required: false + default: ranger + ranger_database: + description: The name of the database to be created for Ranger. + type: str + required: false + default: ranger diff --git a/roles/prereq_ranger_database/molecule/default/converge.yml b/roles/prereq_ranger_database/molecule/default/converge.yml new file mode 100644 index 00000000..7e83bda7 --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Ranger database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_ranger_database diff --git a/roles/prereq_ranger_database/molecule/default/create.yml b/roles/prereq_ranger_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ranger_database/molecule/default/destroy.yml b/roles/prereq_ranger_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ranger_database/molecule/default/molecule.yml b/roles/prereq_ranger_database/molecule/default/molecule.yml new file mode 100644 index 00000000..1da064c7 --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_ranger_database-rhel9-4 + Project: Molecule testing for prereq_ranger_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_ranger_database/molecule/default/prepare.yml b/roles/prereq_ranger_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_ranger_database/molecule/default/requirements.yml b/roles/prereq_ranger_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_ranger_database/molecule/default/verify.yml b/roles/prereq_ranger_database/molecule/default/verify.yml new file mode 100644 index 00000000..4c69d7df --- /dev/null +++ b/roles/prereq_ranger_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'ranger';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database ranger does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'rangeradmin';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User rangeradmin does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_ranger_database/tasks/main.yml b/roles/prereq_ranger_database/tasks/main.yml new file mode 100644 index 00000000..fa9c7a68 --- /dev/null +++ b/roles/prereq_ranger_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ ranger_details }}" + ranger_details: + - user: "{{ ranger_username }}" + password: "{{ ranger_password }}" + db: "{{ ranger_database }}" + no_log: true From f83aeaa55b566fad4aa65bbad174c1992f75e0fe Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:14 -0400 Subject: [PATCH 48/72] Add Reports Manager prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_reportsmanager/README.md | 79 ++++ roles/prereq_reportsmanager/defaults/main.yml | 24 ++ .../meta/argument_specs.yml | 60 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_reportsmanager/tasks/main.yml | 23 ++ 11 files changed, 852 insertions(+) create mode 100644 roles/prereq_reportsmanager/README.md create mode 100644 roles/prereq_reportsmanager/defaults/main.yml create mode 100644 roles/prereq_reportsmanager/meta/argument_specs.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/converge.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/create.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/destroy.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/molecule.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/prepare.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/requirements.yml create mode 100644 roles/prereq_reportsmanager/molecule/default/verify.yml create mode 100644 roles/prereq_reportsmanager/tasks/main.yml diff --git a/roles/prereq_reportsmanager/README.md b/roles/prereq_reportsmanager/README.md new file mode 100644 index 00000000..78faf9f7 --- /dev/null +++ b/roles/prereq_reportsmanager/README.md @@ -0,0 +1,79 @@ +# prereq_reportsmanager + +Set up database and user accounts for Reports Manager + +This role automates the setup of a database and its associated user accounts specifically for Reports Manager services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `reports_manager_database`. +- Create a new database user specified by `reports_manager_username` with the password from `reports_manager_password`. +- Grant ownership and all necessary privileges to the `reports_manager_username` for the new database. +- Ensure the database is configured correctly for Reports Manager operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `reports_manager_username` | `str` | `False` | `rman` | The username for the Reports Manager database user. This user will also be the owner of the database. | +| `reports_manager_password` | `str` | `False` | `rman` | The password for the Reports Manager database user. It is highly recommended to override this default in production. | +| `reports_manager_database` | `str` | `False` | `rman` | The name of the database to be created for Reports Manager. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Reports Manager database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_reportsmanager + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Reports Manager database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_reportsmanager + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + reports_manager_username: "my_rman_user" + reports_manager_password: "a_strong_rman_password" + reports_manager_database: "my_rman_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_reportsmanager/defaults/main.yml b/roles/prereq_reportsmanager/defaults/main.yml new file mode 100644 index 00000000..bfb1e7bf --- /dev/null +++ b/roles/prereq_reportsmanager/defaults/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +reports_manager_username: rman +reports_manager_password: rman +reports_manager_database: rman diff --git a/roles/prereq_reportsmanager/meta/argument_specs.yml b/roles/prereq_reportsmanager/meta/argument_specs.yml new file mode 100644 index 00000000..9af1b5d9 --- /dev/null +++ b/roles/prereq_reportsmanager/meta/argument_specs.yml @@ -0,0 +1,60 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Reports Manager + description: + - Set up the Reports Manager database and its associated user accounts, ensuring proper configuration for Reports Manager operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the `reports_manager_username`, + `reports_manager_password`, and `reports_manager_database` variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + reports_manager_username: + description: The username for the Reports Manager database user and owner of the database. + type: str + required: false + default: rman + reports_manager_password: + description: The password for the Reports Manager database user. + type: str + required: false + default: rman + reports_manager_database: + description: The name of the database to be created for Reports Manager. + type: str + required: false + default: rman diff --git a/roles/prereq_reportsmanager/molecule/default/converge.yml b/roles/prereq_reportsmanager/molecule/default/converge.yml new file mode 100644 index 00000000..9a193352 --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Reports Manager database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_reportsmanager diff --git a/roles/prereq_reportsmanager/molecule/default/create.yml b/roles/prereq_reportsmanager/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_reportsmanager/molecule/default/destroy.yml b/roles/prereq_reportsmanager/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_reportsmanager/molecule/default/molecule.yml b/roles/prereq_reportsmanager/molecule/default/molecule.yml new file mode 100644 index 00000000..6b0b9456 --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_reportsmanager-rhel9-4 + Project: Molecule testing for prereq_reportsmanager +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_reportsmanager/molecule/default/prepare.yml b/roles/prereq_reportsmanager/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_reportsmanager/molecule/default/requirements.yml b/roles/prereq_reportsmanager/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_reportsmanager/molecule/default/verify.yml b/roles/prereq_reportsmanager/molecule/default/verify.yml new file mode 100644 index 00000000..22156138 --- /dev/null +++ b/roles/prereq_reportsmanager/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'rman';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database rman does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'rman';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User rman does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_reportsmanager/tasks/main.yml b/roles/prereq_reportsmanager/tasks/main.yml new file mode 100644 index 00000000..94d5695f --- /dev/null +++ b/roles/prereq_reportsmanager/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ reports_manager_details }}" + reports_manager_details: + - user: "{{ reports_manager_username }}" + password: "{{ reports_manager_password }}" + db: "{{ reports_manager_database }}" From 906fdab4f4a1e67bc3424303323c69a742249ea1 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:15 -0400 Subject: [PATCH 49/72] Add Random Number Generator prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_rngd/README.md | 58 +++ roles/prereq_rngd/defaults/main.yml | 16 + roles/prereq_rngd/handlers/main.yml | 20 ++ roles/prereq_rngd/meta/argument_specs.yml | 23 ++ .../prereq_rngd/molecule/default/converge.yml | 23 ++ roles/prereq_rngd/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_rngd/molecule/default/destroy.yml | 157 ++++++++ .../prereq_rngd/molecule/default/molecule.yml | 53 +++ .../molecule/default/requirements.yml | 18 + roles/prereq_rngd/molecule/default/verify.yml | 43 +++ roles/prereq_rngd/tasks/main.yml | 74 ++++ .../templates/rngd_Ubuntu.service.j2 | 25 ++ roles/prereq_rngd/vars/Debian.yml | 19 + roles/prereq_rngd/vars/RedHat.yml | 17 + roles/prereq_rngd/vars/default.yml | 16 + 15 files changed, 898 insertions(+) create mode 100644 roles/prereq_rngd/README.md create mode 100644 roles/prereq_rngd/defaults/main.yml create mode 100644 roles/prereq_rngd/handlers/main.yml create mode 100644 roles/prereq_rngd/meta/argument_specs.yml create mode 100644 roles/prereq_rngd/molecule/default/converge.yml create mode 100644 roles/prereq_rngd/molecule/default/create.yml create mode 100644 roles/prereq_rngd/molecule/default/destroy.yml create mode 100644 roles/prereq_rngd/molecule/default/molecule.yml create mode 100644 roles/prereq_rngd/molecule/default/requirements.yml create mode 100644 roles/prereq_rngd/molecule/default/verify.yml create mode 100644 roles/prereq_rngd/tasks/main.yml create mode 100644 roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 create mode 100644 roles/prereq_rngd/vars/Debian.yml create mode 100644 roles/prereq_rngd/vars/RedHat.yml create mode 100644 roles/prereq_rngd/vars/default.yml diff --git a/roles/prereq_rngd/README.md b/roles/prereq_rngd/README.md new file mode 100644 index 00000000..a573b84e --- /dev/null +++ b/roles/prereq_rngd/README.md @@ -0,0 +1,58 @@ +# prereq_rngd + +Install the Random Number Generator package + +This role installs and configures the Random Number Generator Daemon (`rngd`) package on a host. The daemon is essential for ensuring that the system has a sufficient amount of entropy available for cryptographic operations, which is crucial for secure communications and services. + +**NOTE** This role should only be used when run on virtual machine (VM); guard role execution by using the conditional `ansible_virtualization_role == 'guest'`. + +The role will: +- Install the `rngd` package using the system's package manager. +- Ensure the `rngd` service is enabled and started, so that it can continuously feed entropy from a hardware random number generator to the kernel's entropy pool. + +# Requirements + +- Root or `sudo` privileges are required on the target host to install and manage system packages and services. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Install and configure the Random Number Generator daemon + ansible.builtin.import_role: + name: cloudera.exe.prereq_rngd + + - name: Install and configure with the virtualization conditional + ansible.builtin.import_role: + name: cloudera.exe.prereq_rngd + when: ansible_virtualization_role == 'guest' +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_rngd/defaults/main.yml b/roles/prereq_rngd/defaults/main.yml new file mode 100644 index 00000000..05affafa --- /dev/null +++ b/roles/prereq_rngd/defaults/main.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# defaults file for prereq_rngd diff --git a/roles/prereq_rngd/handlers/main.yml b/roles/prereq_rngd/handlers/main.yml new file mode 100644 index 00000000..5662cdc6 --- /dev/null +++ b/roles/prereq_rngd/handlers/main.yml @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Restart rngd + ansible.builtin.service: + name: "{{ prereq_rngd__rngd_service }}" + state: restarted + daemon_reload: true diff --git a/roles/prereq_rngd/meta/argument_specs.yml b/roles/prereq_rngd/meta/argument_specs.yml new file mode 100644 index 00000000..5c59fd1f --- /dev/null +++ b/roles/prereq_rngd/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Install the Random Number Generator package" + description: + - Installs and configures the Random Number Generator (rngd) package. + author: + - "Jim Enright " + options: {} diff --git a/roles/prereq_rngd/molecule/default/converge.yml b/roles/prereq_rngd/molecule/default/converge.yml new file mode 100644 index 00000000..b0065eed --- /dev/null +++ b/roles/prereq_rngd/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Run OS rngd prereqs role + ansible.builtin.import_role: + name: cloudera.exe.prereq_rngd diff --git a/roles/prereq_rngd/molecule/default/create.yml b/roles/prereq_rngd/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_rngd/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_rngd/molecule/default/destroy.yml b/roles/prereq_rngd/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_rngd/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_rngd/molecule/default/molecule.yml b/roles/prereq_rngd/molecule/default/molecule.yml new file mode 100644 index 00000000..81d4c0ff --- /dev/null +++ b/roles/prereq_rngd/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_rngd-rhel9-4 + Project: Molecule testing for prereq_rngd + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_rngd-ubuntu20-04 + Project: Molecule testing for prereq_rngd +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_rngd/molecule/default/requirements.yml b/roles/prereq_rngd/molecule/default/requirements.yml new file mode 100644 index 00000000..8ac46513 --- /dev/null +++ b/roles/prereq_rngd/molecule/default/requirements.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - amazon.aws + - ansible.utils diff --git a/roles/prereq_rngd/molecule/default/verify.yml b/roles/prereq_rngd/molecule/default/verify.yml new file mode 100644 index 00000000..0983b593 --- /dev/null +++ b/roles/prereq_rngd/molecule/default/verify.yml @@ -0,0 +1,43 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify + hosts: all + gather_facts: false + tasks: + # Check 1 - confirm that the service present and running + - name: Gather service facts + ansible.builtin.service_facts: + + - name: Set fact for rngd services + ansible.builtin.set_fact: + __rngd_services: "{{ ansible_facts.services | dict2items | selectattr('key', 'search', 'rng') | list }}" + + - name: Confirm that service is present and running + ansible.builtin.assert: + that: + - __rngd_services | length > 0 + - __rngd_services | map(attribute='value') | map(attribute='state') | difference(['running']) | length == 0 + + # Check 2 - confirm that the process is using /dev/urandom + - name: Gather running process details + ansible.builtin.command: ps aux + changed_when: false + register: __rngd_process + + - name: Confirm that rngd process is using /dev/urandom + ansible.builtin.assert: + that: + - __rngd_process.stdout_lines | select('search', 'rngd') | select('contains', '/dev/urandom') | length > 0 diff --git a/roles/prereq_rngd/tasks/main.yml b/roles/prereq_rngd/tasks/main.yml new file mode 100644 index 00000000..8d221605 --- /dev/null +++ b/roles/prereq_rngd/tasks/main.yml @@ -0,0 +1,74 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# NOTE: This role should only be used when run on virtual machine (VM) +# This can be checked with using ansible_virtualization_role == 'guest' + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install rngd + ansible.builtin.package: + name: "{{ prereq_rngd__rngd_package }}" + state: present + update_cache: true + +- name: Enable rngd + ansible.builtin.service: + name: "{{ prereq_rngd__rngd_service }}" + enabled: true + +- name: Configure rngd to use /dev/urandom (RHEL/CentOS 7) + when: + - ansible_os_family == 'RedHat' + - ansible_distribution_major_version|int >= 7 + ansible.builtin.lineinfile: + path: /etc/sysconfig/rngd + regexp: "^RNGD_ARGS=" + line: 'RNGD_ARGS="-r /dev/urandom --fill-watermark=0 -x pkcs11 -x nist -x qrypt -D daemon:daemon"' + backup: true + create: true + mode: "0644" + state: present + notify: Restart rngd + +- name: Configure rngd to use /dev/urandom (Ubuntu) + when: ansible_distribution == 'Ubuntu' + block: + - name: Update rng-tools config file + ansible.builtin.lineinfile: + path: /etc/default/rng-tools + regexp: "^HRNGDEVICE=" + line: "HRNGDEVICE=/dev/urandom" + backup: true + create: true + mode: "0644" + state: present + notify: Restart rngd + + - name: Update the rngd service file + ansible.builtin.template: + src: rngd_Ubuntu.service.j2 + dest: "{{ prereq_rngd__rngd_service_file }}" + mode: "0755" + notify: Restart rngd diff --git a/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 b/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 new file mode 100644 index 00000000..19e1f3dc --- /dev/null +++ b/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 @@ -0,0 +1,25 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Add entropy to /dev/random 's pool a hardware RNG + +[Service] +Type=simple +# Read configuration variable file if it is present +EnvironmentFile=-/etc/default/rng-tools +ExecStart=/usr/sbin/rngd -f -r $HRNGDEVICE $RNGDOPTIONS + +[Install] +WantedBy=dev-hwrng.device \ No newline at end of file diff --git a/roles/prereq_rngd/vars/Debian.yml b/roles/prereq_rngd/vars/Debian.yml new file mode 100644 index 00000000..91a44233 --- /dev/null +++ b/roles/prereq_rngd/vars/Debian.yml @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +prereq_rngd__rngd_package: rng-tools +prereq_rngd__rngd_service: rng-tools +prereq_rngd__rngd_service_file: /lib/systemd/system/rng-tools.service diff --git a/roles/prereq_rngd/vars/RedHat.yml b/roles/prereq_rngd/vars/RedHat.yml new file mode 100644 index 00000000..4334152b --- /dev/null +++ b/roles/prereq_rngd/vars/RedHat.yml @@ -0,0 +1,17 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +prereq_rngd__rngd_package: rng-tools +prereq_rngd__rngd_service: rngd diff --git a/roles/prereq_rngd/vars/default.yml b/roles/prereq_rngd/vars/default.yml new file mode 100644 index 00000000..37e7ce83 --- /dev/null +++ b/roles/prereq_rngd/vars/default.yml @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# vars file for prereq_rngd From ac46f365359712e1b40331ff608a791c17c9d8d4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:16 -0400 Subject: [PATCH 50/72] Add Schema Registry prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_schemaregistry/README.md | 53 +++ .../meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_schemaregistry/tasks/main.yml | 38 ++ roles/prereq_schemaregistry/vars/main.yml | 20 ++ 11 files changed, 794 insertions(+) create mode 100644 roles/prereq_schemaregistry/README.md create mode 100644 roles/prereq_schemaregistry/meta/argument_specs.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/converge.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/create.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/destroy.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/molecule.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/prepare.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/requirements.yml create mode 100644 roles/prereq_schemaregistry/molecule/default/verify.yml create mode 100644 roles/prereq_schemaregistry/tasks/main.yml create mode 100644 roles/prereq_schemaregistry/vars/main.yml diff --git a/roles/prereq_schemaregistry/README.md b/roles/prereq_schemaregistry/README.md new file mode 100644 index 00000000..0073442a --- /dev/null +++ b/roles/prereq_schemaregistry/README.md @@ -0,0 +1,53 @@ +# prereq_schemaregistry + +Set up for Schema Registry + +This role prepares a host for Schema Registry usage by creating a dedicated system user and group named `schemaregistry`. This user is essential for running Schema Registry processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Schema Registry communication. + +The role will: +- Create the `schemaregistry` system user and group. +- Configure home directories and other necessary local paths for the `schemaregistry` user, if required. +- Ensure appropriate permissions are set for files and directories related to Schema Registry. +- Configure TLS ACLs to secure Schema Registry communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: schemaregistry_nodes + tasks: + - name: Set up the schemaregistry user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_schemaregistry +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_schemaregistry/meta/argument_specs.yml b/roles/prereq_schemaregistry/meta/argument_specs.yml new file mode 100644 index 00000000..afaffb0c --- /dev/null +++ b/roles/prereq_schemaregistry/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Schema Registry + description: | + Set up for Schema Registry usage, notably, create the local C(schemaregistry) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_schemaregistry/molecule/default/converge.yml b/roles/prereq_schemaregistry/molecule/default/converge.yml new file mode 100644 index 00000000..54e7ebbe --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Schema Registry + ansible.builtin.import_role: + name: cloudera.exe.prereq_schemaregistry diff --git a/roles/prereq_schemaregistry/molecule/default/create.yml b/roles/prereq_schemaregistry/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_schemaregistry/molecule/default/destroy.yml b/roles/prereq_schemaregistry/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_schemaregistry/molecule/default/molecule.yml b/roles/prereq_schemaregistry/molecule/default/molecule.yml new file mode 100644 index 00000000..0100d92d --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_schemaregistry-rhel9-4 + Project: Molecule testing for prereq_schemaregistry +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_schemaregistry/molecule/default/prepare.yml b/roles/prereq_schemaregistry/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_schemaregistry/molecule/default/requirements.yml b/roles/prereq_schemaregistry/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_schemaregistry/molecule/default/verify.yml b/roles/prereq_schemaregistry/molecule/default/verify.yml new file mode 100644 index 00000000..5802ac65 --- /dev/null +++ b/roles/prereq_schemaregistry/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for Schema Registry users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ schemaregistry_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ schemaregistry_local_accounts }}" diff --git a/roles/prereq_schemaregistry/tasks/main.yml b/roles/prereq_schemaregistry/tasks/main.yml new file mode 100644 index 00000000..df675f1d --- /dev/null +++ b/roles/prereq_schemaregistry/tasks/main.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ schemaregistry_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ schemaregistry_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_schemaregistry/vars/main.yml b/roles/prereq_schemaregistry/vars/main.yml new file mode 100644 index 00000000..8002b052 --- /dev/null +++ b/roles/prereq_schemaregistry/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +schemaregistry_local_accounts: + - user: schemaregistry + home: /var/lib/schemaregistry + comment: Schemaregistry + keystore_acl: true From 7cfb8b4a952dd68c3ca7e05384814e5c9853ecd4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:16 -0400 Subject: [PATCH 51/72] Add Schema Registry database prerequisites role Signed-off-by: Webster Mudge --- .../prereq_schemaregistry_database/README.md | 80 +++++ .../defaults/main.yml | 25 ++ .../meta/argument_specs.yml | 62 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ .../tasks/main.yml | 24 ++ 11 files changed, 857 insertions(+) create mode 100644 roles/prereq_schemaregistry_database/README.md create mode 100644 roles/prereq_schemaregistry_database/defaults/main.yml create mode 100644 roles/prereq_schemaregistry_database/meta/argument_specs.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/converge.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/create.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/destroy.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/molecule.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/prepare.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/requirements.yml create mode 100644 roles/prereq_schemaregistry_database/molecule/default/verify.yml create mode 100644 roles/prereq_schemaregistry_database/tasks/main.yml diff --git a/roles/prereq_schemaregistry_database/README.md b/roles/prereq_schemaregistry_database/README.md new file mode 100644 index 00000000..9c83aa3a --- /dev/null +++ b/roles/prereq_schemaregistry_database/README.md @@ -0,0 +1,80 @@ +# prereq_schemaregistry_database + +Set up database and user accounts for Schema Registry + +This role automates the setup of a database and its associated user accounts specifically for Schema Registry services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `schemaregistry_database`. +- Create a new database user specified by `schemaregistry_username` with the password from `schemaregistry_password`. +- Grant ownership and all necessary privileges to the `schemaregistry_username` for the new database. +- Ensure the database is configured correctly for Schema Registry operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_port` | `int` | `False` | - | The port for connecting to the database server. If not specified, the role will use the default port for the specified database type. | +| `schemaregistry_username` | `str` | `False` | `registry` | The username for the Schema Registry database user. This user will also be the owner of the database. | +| `schemaregistry_password` | `str` | `False` | `registry` | The password for the Schema Registry database user. It is highly recommended to override this default in production. | +| `schemaregistry_database` | `str` | `False` | `registry` | The name of the database to be created for Schema Registry. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up Schema Registry database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_schemaregistry_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up Schema Registry database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_schemaregistry_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + schemaregistry_username: "my_sr_user" + schemaregistry_password: "a_strong_sr_password" + schemaregistry_database: "my_sr_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_schemaregistry_database/defaults/main.yml b/roles/prereq_schemaregistry_database/defaults/main.yml new file mode 100644 index 00000000..073fcdde --- /dev/null +++ b/roles/prereq_schemaregistry_database/defaults/main.yml @@ -0,0 +1,25 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +database_type: "{{ undef(hint='Please define the database type.') }}" +database_host: "{{ undef(hint='Please define the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please define the administrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please define the administrator password for the database server.') }}" + +schemaregistry_username: registry +schemaregistry_password: registry +schemaregistry_database: registry diff --git a/roles/prereq_schemaregistry_database/meta/argument_specs.yml b/roles/prereq_schemaregistry_database/meta/argument_specs.yml new file mode 100644 index 00000000..7236f747 --- /dev/null +++ b/roles/prereq_schemaregistry_database/meta/argument_specs.yml @@ -0,0 +1,62 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Schema Registry + description: | + This role manages the creation of the database and its associated user accounts for Schema Registry. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_admin_user: + description: The username for the database admin login. + type: str + required: true + database_admin_password: + description: The password for the database admin login. + type: str + required: true + no_log: true + database_host: + description: The hostname or IP address of the database server. + type: str + required: true + database_port: + description: The port for connecting to the database server. + type: int + required: false + schemaregistry_username: + description: The username for the Schema Registry database user and owner of the database. + type: str + required: false + default: registry + schemaregistry_password: + description: The password for the Schema Registry database user. + type: str + required: false + default: registry + schemaregistry_database: + description: The name of the database to be created for Schema Registry. + type: str + required: false + default: registry diff --git a/roles/prereq_schemaregistry_database/molecule/default/converge.yml b/roles/prereq_schemaregistry_database/molecule/default/converge.yml new file mode 100644 index 00000000..19cd19e7 --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create Schema Registry database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_schemaregistry_database diff --git a/roles/prereq_schemaregistry_database/molecule/default/create.yml b/roles/prereq_schemaregistry_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_schemaregistry_database/molecule/default/destroy.yml b/roles/prereq_schemaregistry_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_schemaregistry_database/molecule/default/molecule.yml b/roles/prereq_schemaregistry_database/molecule/default/molecule.yml new file mode 100644 index 00000000..2384e2ed --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_schemaregistry_database-rhel9-4 + Project: Molecule testing for prereq_schemaregistry_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_schemaregistry_database/molecule/default/prepare.yml b/roles/prereq_schemaregistry_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_schemaregistry_database/molecule/default/requirements.yml b/roles/prereq_schemaregistry_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_schemaregistry_database/molecule/default/verify.yml b/roles/prereq_schemaregistry_database/molecule/default/verify.yml new file mode 100644 index 00000000..8911ca32 --- /dev/null +++ b/roles/prereq_schemaregistry_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'registry';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database registry does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'registry';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User registry does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_schemaregistry_database/tasks/main.yml b/roles/prereq_schemaregistry_database/tasks/main.yml new file mode 100644 index 00000000..4a4d4b34 --- /dev/null +++ b/roles/prereq_schemaregistry_database/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ schemaregistry_details }}" + schemaregistry_details: + - user: "{{ schemaregistry_username }}" + password: "{{ schemaregistry_password }}" + db: "{{ schemaregistry_database }}" + no_log: true From 21018f9f1fa148edfaef7b0d2ab12e5140fdcf94 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:17 -0400 Subject: [PATCH 52/72] Add SELinux prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_selinux/README.md | 69 ++++ roles/prereq_selinux/defaults/main.yml | 17 + roles/prereq_selinux/handlers/main.yml | 17 + roles/prereq_selinux/meta/argument_specs.yml | 39 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 53 +++ .../molecule/default/prepare.yml | 29 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 24 ++ roles/prereq_selinux/tasks/main.yml | 45 +++ roles/prereq_selinux/vars/RedHat.yml | 21 ++ roles/prereq_selinux/vars/Ubuntu.yml | 24 ++ roles/prereq_selinux/vars/default.yml | 21 ++ 15 files changed, 896 insertions(+) create mode 100644 roles/prereq_selinux/README.md create mode 100644 roles/prereq_selinux/defaults/main.yml create mode 100644 roles/prereq_selinux/handlers/main.yml create mode 100644 roles/prereq_selinux/meta/argument_specs.yml create mode 100644 roles/prereq_selinux/molecule/default/converge.yml create mode 100644 roles/prereq_selinux/molecule/default/create.yml create mode 100644 roles/prereq_selinux/molecule/default/destroy.yml create mode 100644 roles/prereq_selinux/molecule/default/molecule.yml create mode 100644 roles/prereq_selinux/molecule/default/prepare.yml create mode 100644 roles/prereq_selinux/molecule/default/requirements.yml create mode 100644 roles/prereq_selinux/molecule/default/verify.yml create mode 100644 roles/prereq_selinux/tasks/main.yml create mode 100644 roles/prereq_selinux/vars/RedHat.yml create mode 100644 roles/prereq_selinux/vars/Ubuntu.yml create mode 100644 roles/prereq_selinux/vars/default.yml diff --git a/roles/prereq_selinux/README.md b/roles/prereq_selinux/README.md new file mode 100644 index 00000000..9bc84a20 --- /dev/null +++ b/roles/prereq_selinux/README.md @@ -0,0 +1,69 @@ +# prereq_selinux + +Manage SELinux policy enforcement + +This role is designed to manage the enforcement state and policy of SELinux on a host. It can set SELinux to `enforcing`, `permissive`, or `disabled` mode. The role ensures that the necessary packages for SELinux management are installed, but it assumes that SELinux itself is already part of the system's kernel. + +The role will: +- Install packages required for SELinux administration (e.g., `selinux-policy`, `libselinux-python`). +- Configure the SELinux state (`selinux_state`) to `disabled`, `enforcing`, or `permissive`. +- Set the SELinux policy (`selinux_policy`), with OS-specific defaults if not specified. +- Apply the changes to the system to take effect immediately and persist across reboots. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage SELinux. +- The system kernel must have SELinux support compiled in and enabled for the role to have its full effect. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `selinux_state` | `str` | `False` | `permissive` | The desired enforcement mode for SELinux. `disabled` completely turns off SELinux, `permissive` logs but does not enforce policy, and `enforcing` enforces policy and logs denials. Valid choices are `disabled`, `enforcing`, `permissive`. | +| `selinux_policy` | `str` | `False` | - | The policy to adopt for SELinux. On Red Hat-based distributions, the default is `targeted`. On Ubuntu-based distributions, the default is `default`. This variable should be set to match the desired policy file name. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Set SELinux to permissive mode + ansible.builtin.import_role: + name: cloudera.exe.prereq_selinux + # This will install SELinux management tools and set the state to permissive by default. + + - name: Set SELinux to enforcing mode with the targeted policy + ansible.builtin.import_role: + name: cloudera.exe.prereq_selinux + vars: + selinux_state: enforcing + selinux_policy: targeted + + - name: Disable SELinux entirely + ansible.builtin.import_role: + name: cloudera.exe.prereq_selinux + vars: + selinux_state: disabled +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/roles/prereq_selinux/defaults/main.yml b/roles/prereq_selinux/defaults/main.yml new file mode 100644 index 00000000..c0f79d04 --- /dev/null +++ b/roles/prereq_selinux/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +selinux_state: permissive +# selinux_policy: diff --git a/roles/prereq_selinux/handlers/main.yml b/roles/prereq_selinux/handlers/main.yml new file mode 100644 index 00000000..5e537b46 --- /dev/null +++ b/roles/prereq_selinux/handlers/main.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Reboot host + ansible.builtin.reboot: diff --git a/roles/prereq_selinux/meta/argument_specs.yml b/roles/prereq_selinux/meta/argument_specs.yml new file mode 100644 index 00000000..98833ccd --- /dev/null +++ b/roles/prereq_selinux/meta/argument_specs.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Manage SELinux policy enforcement. + description: + - Manage the enforcement of the SELinux. + - Installs packages for SELinux management, if needed, but does not install SELinux. + author: + - Webster Mudge (wmudge@cloudera.com) + options: + selinux_state: + description: + - The mode for SELinux. + type: str + default: permissive + choices: + - disabled + - enforcing + - permissive + selinux_policy: + description: + - The policy to adopt for SELinux. + - Default (RedHat) - C(targeted). + - Default (Ubuntu) - C(default). + type: str diff --git a/roles/prereq_selinux/molecule/default/converge.yml b/roles/prereq_selinux/molecule/default/converge.yml new file mode 100644 index 00000000..f431d07f --- /dev/null +++ b/roles/prereq_selinux/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Set SELinux to 'permissive' state + ansible.builtin.import_role: + name: cloudera.exe.prereq_selinux diff --git a/roles/prereq_selinux/molecule/default/create.yml b/roles/prereq_selinux/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_selinux/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_selinux/molecule/default/destroy.yml b/roles/prereq_selinux/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_selinux/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_selinux/molecule/default/molecule.yml b/roles/prereq_selinux/molecule/default/molecule.yml new file mode 100644 index 00000000..a162bf44 --- /dev/null +++ b/roles/prereq_selinux/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_selinux-rhel9-4 + Project: Molecule testing for prereq_selinux + # Ubuntu 20.04 + - name: ubuntu20_04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_selinux-ubuntu20-04 + Project: Molecule testing for prereq_selinux +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_selinux/molecule/default/prepare.yml b/roles/prereq_selinux/molecule/default/prepare.yml new file mode 100644 index 00000000..eb210ad9 --- /dev/null +++ b/roles/prereq_selinux/molecule/default/prepare.yml @@ -0,0 +1,29 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: true + become: true + tasks: + - name: Update package cache (apt) + when: ansible_pkg_mgr == "apt" + ansible.builtin.apt: + update_cache: true + + - name: Update package cache (dnf) + when: ansible_pkg_mgr == "dnf" + ansible.builtin.dnf: + update_cache: true diff --git a/roles/prereq_selinux/molecule/default/requirements.yml b/roles/prereq_selinux/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_selinux/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_selinux/molecule/default/verify.yml b/roles/prereq_selinux/molecule/default/verify.yml new file mode 100644 index 00000000..d3f9ff63 --- /dev/null +++ b/roles/prereq_selinux/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all:!ubuntu20_04.molecule.internal + gather_facts: false + tasks: + - name: Check SELinux state + ansible.builtin.command: getenforce + register: __selinux + changed_when: false + failed_when: __selinux.stdout is not search("Permissive") diff --git a/roles/prereq_selinux/tasks/main.yml b/roles/prereq_selinux/tasks/main.yml new file mode 100644 index 00000000..8fd116ec --- /dev/null +++ b/roles/prereq_selinux/tasks/main.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Install SELinux management dependencies + ansible.builtin.package: + name: "{{ selinux_packages }}" + state: present + +- name: Manage SELinux policy state + ansible.posix.selinux: + policy: "{{ selinux_policy | default(__selinux_policy) }}" + state: "{{ selinux_state }}" + notify: Reboot host + +- name: Persist SELinux state + ansible.builtin.lineinfile: + path: "{{ selinux_config_file }}" + regexp: "^(#)?SELINUX=" + line: "SELINUX={{ selinux_state }}" + mode: "0644" + state: present + notify: Reboot host diff --git a/roles/prereq_selinux/vars/RedHat.yml b/roles/prereq_selinux/vars/RedHat.yml new file mode 100644 index 00000000..3dc53a56 --- /dev/null +++ b/roles/prereq_selinux/vars/RedHat.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +selinux_packages: + - python3-libselinux + +__selinux_policy: targeted + +selinux_config_file: /etc/selinux/config diff --git a/roles/prereq_selinux/vars/Ubuntu.yml b/roles/prereq_selinux/vars/Ubuntu.yml new file mode 100644 index 00000000..d0d5bc59 --- /dev/null +++ b/roles/prereq_selinux/vars/Ubuntu.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +selinux_packages: + - selinux-basics + - selinux-utils + - selinux-policy-default + - python3-selinux + +__selinux_policy: default + +selinux_config_file: /etc/selinux/config diff --git a/roles/prereq_selinux/vars/default.yml b/roles/prereq_selinux/vars/default.yml new file mode 100644 index 00000000..a8e199e6 --- /dev/null +++ b/roles/prereq_selinux/vars/default.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +selinux_packages: + - python3-libselinux + +__selinux_policy: default + +selinux_config_file: /etc/selinux/config From 3b5de012c86807613a23186ad2f7317cc4f607e7 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:18 -0400 Subject: [PATCH 53/72] Add Apache Sentry prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_sentry/README.md | 52 +++ roles/prereq_sentry/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_sentry/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ .../prereq_sentry/molecule/default/verify.yml | 24 ++ roles/prereq_sentry/tasks/main.yml | 39 ++ roles/prereq_sentry/vars/main.yml | 22 ++ 10 files changed, 738 insertions(+) create mode 100644 roles/prereq_sentry/README.md create mode 100644 roles/prereq_sentry/meta/argument_specs.yml create mode 100644 roles/prereq_sentry/molecule/default/converge.yml create mode 100644 roles/prereq_sentry/molecule/default/create.yml create mode 100644 roles/prereq_sentry/molecule/default/destroy.yml create mode 100644 roles/prereq_sentry/molecule/default/molecule.yml create mode 100644 roles/prereq_sentry/molecule/default/requirements.yml create mode 100644 roles/prereq_sentry/molecule/default/verify.yml create mode 100644 roles/prereq_sentry/tasks/main.yml create mode 100644 roles/prereq_sentry/vars/main.yml diff --git a/roles/prereq_sentry/README.md b/roles/prereq_sentry/README.md new file mode 100644 index 00000000..668b4a48 --- /dev/null +++ b/roles/prereq_sentry/README.md @@ -0,0 +1,52 @@ +# prereq_sentry + +Set up for Sentry + +This role prepares a host for Apache Sentry usage by creating a dedicated system user and group named `sentry`. This user is essential for running Sentry processes with appropriate permissions and isolation. + +The role will: +- Create the `sentry` system user and group. +- Configure home directories and other necessary local paths for the `sentry` user, if required. +- Ensure appropriate permissions are set for files and directories related to Sentry. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: sentry_nodes + tasks: + - name: Set up the sentry user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_sentry +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_sentry/meta/argument_specs.yml b/roles/prereq_sentry/meta/argument_specs.yml new file mode 100644 index 00000000..77e5f1ea --- /dev/null +++ b/roles/prereq_sentry/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Sentry + description: | + Set up for Apache Sentry usage, notably, create the local C(sentry) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_sentry/molecule/default/converge.yml b/roles/prereq_sentry/molecule/default/converge.yml new file mode 100644 index 00000000..b052bad6 --- /dev/null +++ b/roles/prereq_sentry/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Sentry + ansible.builtin.import_role: + name: cloudera.exe.prereq_sentry diff --git a/roles/prereq_sentry/molecule/default/create.yml b/roles/prereq_sentry/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_sentry/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_sentry/molecule/default/destroy.yml b/roles/prereq_sentry/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_sentry/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_sentry/molecule/default/molecule.yml b/roles/prereq_sentry/molecule/default/molecule.yml new file mode 100644 index 00000000..b30fd02c --- /dev/null +++ b/roles/prereq_sentry/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_sentry-rhel9-4 + Project: Molecule testing for prereq_sentry +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_sentry/molecule/default/requirements.yml b/roles/prereq_sentry/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_sentry/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_sentry/molecule/default/verify.yml b/roles/prereq_sentry/molecule/default/verify.yml new file mode 100644 index 00000000..459a0fa8 --- /dev/null +++ b/roles/prereq_sentry/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check for sentry user + ansible.builtin.command: grep sentry /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false diff --git a/roles/prereq_sentry/tasks/main.yml b/roles/prereq_sentry/tasks/main.yml new file mode 100644 index 00000000..cc8136c7 --- /dev/null +++ b/roles/prereq_sentry/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.user: + name: "{{ account.user }}" + home: "{{ account.home }}" + comment: "{{ account.comment | default(omit) }}" + groups: "{{ account.extra_groups | default(omit) }}" + append: true + uid: "{{ account.uid | default(omit) }}" + shell: "{{ account.shell | default(sentry_default_shell) }}" + loop: "{{ sentry_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Set local user home directory permissions + ansible.builtin.file: + path: "{{ account.home }}" + owner: "{{ account.user }}" + group: "{{ account.user }}" + mode: "{{ account.mode | default(sentry_default_home_dir_mode) }}" + loop: "{{ sentry_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.home }}" diff --git a/roles/prereq_sentry/vars/main.yml b/roles/prereq_sentry/vars/main.yml new file mode 100644 index 00000000..ecfd0770 --- /dev/null +++ b/roles/prereq_sentry/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sentry_local_accounts: + - user: sentry + home: /var/lib/sentry + comment: sentry + +sentry_default_shell: /sbin/nologin +sentry_default_home_dir_mode: "0755" From 9589cd529375ade7bfc60249375d6a9b1af5a292 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:19 -0400 Subject: [PATCH 54/72] Add general system services prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_services/README.md | 63 ++++ roles/prereq_services/defaults/main.yml | 23 ++ roles/prereq_services/handlers/main.yml | 21 ++ roles/prereq_services/meta/argument_specs.yml | 40 +++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 53 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 44 +++ roles/prereq_services/tasks/main.yml | 58 +++ 11 files changed, 839 insertions(+) create mode 100644 roles/prereq_services/README.md create mode 100644 roles/prereq_services/defaults/main.yml create mode 100644 roles/prereq_services/handlers/main.yml create mode 100644 roles/prereq_services/meta/argument_specs.yml create mode 100644 roles/prereq_services/molecule/default/converge.yml create mode 100644 roles/prereq_services/molecule/default/create.yml create mode 100644 roles/prereq_services/molecule/default/destroy.yml create mode 100644 roles/prereq_services/molecule/default/molecule.yml create mode 100644 roles/prereq_services/molecule/default/requirements.yml create mode 100644 roles/prereq_services/molecule/default/verify.yml create mode 100644 roles/prereq_services/tasks/main.yml diff --git a/roles/prereq_services/README.md b/roles/prereq_services/README.md new file mode 100644 index 00000000..c708c3f1 --- /dev/null +++ b/roles/prereq_services/README.md @@ -0,0 +1,63 @@ +# prereq_services + +Manage operating system services + +This role is designed to manage operating system services to meet the specific requirements of a Cloudera on-premises deployment. It takes a proactive approach by disabling services that are typically unnecessary in a cluster environment and ensuring that essential services, such as the Name Service Cache Daemon (NSCD), are installed, configured, and running. + +The role will: +- Install the `nscd` package and ensure the corresponding service is enabled and started. +- Iterate through a list of unnecessary services, stopping and disabling each one to reduce system overhead and potential security risks. +- Ensure all services are in the correct state (either running and enabled, or stopped and disabled) to match the desired configuration. + +# Requirements + +- Root or `sudo` privileges are required on the target host to manage system packages and services. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `prereq_services_unnecessary_services` | `list` of `str` | `False` | `["bluetooth", "cups", "postfix"]` | A list of OS service names that will be stopped and disabled. | +| `prereq_services_nscd_package` | `str` | `False` | `nscd` | The name of the package for the Name Service Cache Daemon (NSCD) to install. This may vary between OS distributions. | +| `prereq_services_nscd_service` | `str` | `False` | `nscd` | The name of the NSCD service to enable and start. This may vary between OS distributions. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Manage default OS services for Cloudera + ansible.builtin.import_role: + name: cloudera.exe.prereq_services + # This will install nscd and disable bluetooth, cups, and postfix. + + - name: Manage OS services with a custom list of unnecessary services + ansible.builtin.import_role: + name: cloudera.exe.prereq_services + vars: + prereq_services__unnecessary_services: + - avahi-daemon + - auditd +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_services/defaults/main.yml b/roles/prereq_services/defaults/main.yml new file mode 100644 index 00000000..46c8fcf4 --- /dev/null +++ b/roles/prereq_services/defaults/main.yml @@ -0,0 +1,23 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +prereq_services_unnecessary_services: + - bluetooth + - cups + - postfix + +prereq_services_nscd_package: nscd +prereq_services_nscd_service: nscd diff --git a/roles/prereq_services/handlers/main.yml b/roles/prereq_services/handlers/main.yml new file mode 100644 index 00000000..905df187 --- /dev/null +++ b/roles/prereq_services/handlers/main.yml @@ -0,0 +1,21 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# handlers file for prereq_services + +- name: Restart nscd + ansible.builtin.service: + name: "{{ prereq_services__nscd_service }}" + state: restarted diff --git a/roles/prereq_services/meta/argument_specs.yml b/roles/prereq_services/meta/argument_specs.yml new file mode 100644 index 00000000..d5f3a1ec --- /dev/null +++ b/roles/prereq_services/meta/argument_specs.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Manage operating system services" + description: + - Manage operating system services for Cloudera on premises deployment. + - Includes installing and configuring required services as well as disabling unnecessary services. + author: + - "Jim Enright " + options: + prereq_services_unnecessary_services: + description: "List of unwanted OS services that will be disabled." + type: "list" + elements: "str" + default: + - bluetooth + - cups + - postfix + prereq_services_nscd_package: + description: Name of the nscd package to install + type: "str" + default: "nscd" + prereq_services_nscd_service: + description: Name of the nscd service to enable + type: "str" + default: "nscd" diff --git a/roles/prereq_services/molecule/default/converge.yml b/roles/prereq_services/molecule/default/converge.yml new file mode 100644 index 00000000..287b9da3 --- /dev/null +++ b/roles/prereq_services/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Run OS services prereqs role + ansible.builtin.import_role: + name: cloudera.exe.prereq_services diff --git a/roles/prereq_services/molecule/default/create.yml b/roles/prereq_services/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_services/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_services/molecule/default/destroy.yml b/roles/prereq_services/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_services/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_services/molecule/default/molecule.yml b/roles/prereq_services/molecule/default/molecule.yml new file mode 100644 index 00000000..6bd8eba0 --- /dev/null +++ b/roles/prereq_services/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_services-rhel9-4 + Project: Molecule testing for prereq_services + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_services-ubuntu20-04 + Project: Molecule testing for prereq_services +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_services/molecule/default/requirements.yml b/roles/prereq_services/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_services/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_services/molecule/default/verify.yml b/roles/prereq_services/molecule/default/verify.yml new file mode 100644 index 00000000..7d4de680 --- /dev/null +++ b/roles/prereq_services/molecule/default/verify.yml @@ -0,0 +1,44 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify + hosts: all + gather_facts: true + vars: + unnecessary_services: + - bluetooth + - cups + - postfix + tasks: + - name: Populate service facts + ansible.builtin.service_facts: + + # Check 1 - confirm that the unwanted services are stopped + - name: Confirm that unnecessary services are stopped + when: "__service + '.service' in ansible_facts.services" + ansible.builtin.assert: + that: + - ansible_facts.services[__service + '.service'].state != 'running' + fail_msg: "Services is in running state. Should be stopped." + loop_control: + loop_var: __service + loop: "{{ unnecessary_services }}" + + # Check 2 - confirm that nscd is running + - name: Confirm that nscd service is running + ansible.builtin.assert: + that: + - ansible_facts.services['nscd.service'].state == 'running' + fail_msg: "ncsd services should be in running state. Currently '{{ ansible_facts.services['nscd.service'].state }}'." diff --git a/roles/prereq_services/tasks/main.yml b/roles/prereq_services/tasks/main.yml new file mode 100644 index 00000000..0efcf7b9 --- /dev/null +++ b/roles/prereq_services/tasks/main.yml @@ -0,0 +1,58 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Disable unwanted services +- name: Populate service facts + ansible.builtin.service_facts: + +- name: Disable unnecessary services + when: "__service + '.service' in ansible_facts.services" + ansible.builtin.service: + name: "{{ __service }}" + state: stopped + enabled: false + loop_control: + loop_var: __service + loop: "{{ prereq_services_unnecessary_services }}" + +# Enable and configure nscd service +- name: Configure universe repository for Debian OS + when: ansible_facts['os_family'] == "Debian" + block: + - name: Enable universe repository + ansible.builtin.apt_repository: + repo: "deb http://archive.ubuntu.com/ubuntu {{ ansible_distribution_release }} universe" + state: present + update_cache: true + +- name: Install nscd service + ansible.builtin.package: + lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}" + name: "{{ prereq_services_nscd_package }}" + state: present + +- name: Enable nscd service + ansible.builtin.service: + name: "{{ prereq_services_nscd_service }}" + state: started + enabled: true + +- name: Disable nscd caches for services 'passwd', 'group', 'netgroup' + ansible.builtin.replace: + path: /etc/nscd.conf + regexp: "^(.*enable-cache.*(passwd|group|netgroup).*)yes$" + replace: "\\1no" + notify: + - Restart nscd From c95e3a974a981e5bfe731db7483a4b613f8a7c9f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:19 -0400 Subject: [PATCH 55/72] Add Streams Messaging Manager prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_smm/README.md | 53 +++ roles/prereq_smm/meta/argument_specs.yml | 23 ++ .../prereq_smm/molecule/default/converge.yml | 23 ++ roles/prereq_smm/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_smm/molecule/default/destroy.yml | 157 ++++++++ .../prereq_smm/molecule/default/molecule.yml | 49 +++ roles/prereq_smm/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_smm/molecule/default/verify.yml | 47 +++ roles/prereq_smm/tasks/main.yml | 50 +++ roles/prereq_smm/vars/main.yml | 26 ++ 11 files changed, 823 insertions(+) create mode 100644 roles/prereq_smm/README.md create mode 100644 roles/prereq_smm/meta/argument_specs.yml create mode 100644 roles/prereq_smm/molecule/default/converge.yml create mode 100644 roles/prereq_smm/molecule/default/create.yml create mode 100644 roles/prereq_smm/molecule/default/destroy.yml create mode 100644 roles/prereq_smm/molecule/default/molecule.yml create mode 100644 roles/prereq_smm/molecule/default/prepare.yml create mode 100644 roles/prereq_smm/molecule/default/requirements.yml create mode 100644 roles/prereq_smm/molecule/default/verify.yml create mode 100644 roles/prereq_smm/tasks/main.yml create mode 100644 roles/prereq_smm/vars/main.yml diff --git a/roles/prereq_smm/README.md b/roles/prereq_smm/README.md new file mode 100644 index 00000000..9813d18a --- /dev/null +++ b/roles/prereq_smm/README.md @@ -0,0 +1,53 @@ +# prereq_smm + +Set up for Streams Messaging Manager + +This role prepares a host for Streams Messaging Manager (SMM) usage by creating dedicated system users and groups for SMM and Streams Replication Manager. It specifically creates the `streamsmsgmgr` and `streamsrepmgr` users. Additionally, it creates a symbolic link for the `streamsmsgmgr` user's home directory, which is a required step for the Custom Service Descriptor (CSD) installation. + +The role will: +- Create the `streamsmsgmgr` and `streamsrepmgr` system users and groups. +- Configure home directories and other necessary local paths for these users. +- Ensure appropriate permissions are set for files and directories. +- Create a symbolic link for the `streamsmsgmgr` user's home directory to fulfill CSD installation requirements. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users, groups, and symbolic links. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: smm_nodes + tasks: + - name: Set up the SMM users and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_smm +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_smm/meta/argument_specs.yml b/roles/prereq_smm/meta/argument_specs.yml new file mode 100644 index 00000000..fb8b555b --- /dev/null +++ b/roles/prereq_smm/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Streams Messaging Manager + description: + - Set up for Streams Messaging Manager usage, notably, create the local C(streamsmsgmgr) and C(streamsrepmgr) users. + - Also creates a symbolic link of the Streams Messaging Manager user home directory required for the Custom Service Descriptor (CSD) installation. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_smm/molecule/default/converge.yml b/roles/prereq_smm/molecule/default/converge.yml new file mode 100644 index 00000000..f64f2db9 --- /dev/null +++ b/roles/prereq_smm/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Streams Messaging Manager + ansible.builtin.import_role: + name: cloudera.exe.prereq_smm diff --git a/roles/prereq_smm/molecule/default/create.yml b/roles/prereq_smm/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_smm/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_smm/molecule/default/destroy.yml b/roles/prereq_smm/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_smm/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_smm/molecule/default/molecule.yml b/roles/prereq_smm/molecule/default/molecule.yml new file mode 100644 index 00000000..9b24d287 --- /dev/null +++ b/roles/prereq_smm/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_smm-rhel9-4 + Project: Molecule testing for prereq_smm +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_smm/molecule/default/prepare.yml b/roles/prereq_smm/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_smm/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_smm/molecule/default/requirements.yml b/roles/prereq_smm/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_smm/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_smm/molecule/default/verify.yml b/roles/prereq_smm/molecule/default/verify.yml new file mode 100644 index 00000000..2faad8a5 --- /dev/null +++ b/roles/prereq_smm/molecule/default/verify.yml @@ -0,0 +1,47 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for smm users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ smm_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Confirm symlinks exists + when: account.symlink is defined + ansible.builtin.stat: + path: "{{ account.symlink }}" + register: _symlink + failed_when: not _symlink.stat.islnk + loop: "{{ smm_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ smm_local_accounts }}" diff --git a/roles/prereq_smm/tasks/main.yml b/roles/prereq_smm/tasks/main.yml new file mode 100644 index 00000000..cec10336 --- /dev/null +++ b/roles/prereq_smm/tasks/main.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ smm_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ smm_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl + +- name: Create symlink for SMM user's home directory + when: account.symlink is defined + ansible.builtin.file: + src: "{{ account.home }}" + dest: "{{ account.symlink }}" + state: link + loop: "{{ smm_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" diff --git a/roles/prereq_smm/vars/main.yml b/roles/prereq_smm/vars/main.yml new file mode 100644 index 00000000..4e8420de --- /dev/null +++ b/roles/prereq_smm/vars/main.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +smm_local_accounts: + - user: streamsmsgmgr + home: /var/lib/streams_messaging_manager + symlink: /var/lib/streamsmsgmgr + comment: Streams Messaging Manager + keystore_acl: true + key_acl: true + - user: streamsrepmgr + home: /var/lib/streams_replication_manager + comment: Streams Replication Manager + keystore_acl: true From 54704c841e185493737698428b0731f6a438a1b5 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:20 -0400 Subject: [PATCH 56/72] Add Streams Messaging Manager database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_smm_database/README.md | 79 +++++ roles/prereq_smm_database/defaults/main.yml | 25 ++ .../meta/argument_specs.yml | 58 +++ .../molecule/default/converge.yml | 22 ++ .../molecule/default/create.yml | 335 ++++++++++++++++++ .../molecule/default/destroy.yml | 156 ++++++++ .../molecule/default/molecule.yml | 45 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_smm_database/tasks/main.yml | 27 ++ 11 files changed, 851 insertions(+) create mode 100644 roles/prereq_smm_database/README.md create mode 100644 roles/prereq_smm_database/defaults/main.yml create mode 100644 roles/prereq_smm_database/meta/argument_specs.yml create mode 100644 roles/prereq_smm_database/molecule/default/converge.yml create mode 100644 roles/prereq_smm_database/molecule/default/create.yml create mode 100644 roles/prereq_smm_database/molecule/default/destroy.yml create mode 100644 roles/prereq_smm_database/molecule/default/molecule.yml create mode 100644 roles/prereq_smm_database/molecule/default/prepare.yml create mode 100644 roles/prereq_smm_database/molecule/default/requirements.yml create mode 100644 roles/prereq_smm_database/molecule/default/verify.yml create mode 100644 roles/prereq_smm_database/tasks/main.yml diff --git a/roles/prereq_smm_database/README.md b/roles/prereq_smm_database/README.md new file mode 100644 index 00000000..a8f551a6 --- /dev/null +++ b/roles/prereq_smm_database/README.md @@ -0,0 +1,79 @@ +# prereq_smm_database + +Set up database and user accounts for Streams Messaging Manager + +This role automates the setup of a database and its associated user accounts specifically for Streams Messaging Manager (SMM) services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- Create a new database with the name specified by `smm_database`. +- Create a new database user specified by `smm_username` with the password from `smm_password`. +- Grant ownership and all necessary privileges to the `smm_username` for the new database. +- Ensure the database is configured correctly for Streams Messaging Manager operations. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `smm_username` | `str` | `False` | `streamsmsgmgr` | The username for the Streams Messaging Manager database user. This user will also be the owner of the database. | +| `smm_password` | `str` | `False` | `streamsmsgmgr` | The password for the Streams Messaging Manager database user. It is highly recommended to override this default in production. | +| `smm_database` | `str` | `False` | `streamsmsgmgr` | The name of the database to be created for Streams Messaging Manager. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up SMM database and user on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_smm_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up SMM database with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_smm_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + smm_username: "my_smm_user" + smm_password: "a_strong_smm_password" + smm_database: "my_smm_db" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_smm_database/defaults/main.yml b/roles/prereq_smm_database/defaults/main.yml new file mode 100644 index 00000000..7f237e25 --- /dev/null +++ b/roles/prereq_smm_database/defaults/main.yml @@ -0,0 +1,25 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +smm_username: streamsmsgmgr +smm_password: streamsmsgmgr +smm_database: streamsmsgmgr diff --git a/roles/prereq_smm_database/meta/argument_specs.yml b/roles/prereq_smm_database/meta/argument_specs.yml new file mode 100644 index 00000000..6feb5abd --- /dev/null +++ b/roles/prereq_smm_database/meta/argument_specs.yml @@ -0,0 +1,58 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for Streams Messaging Manager + description: + - Set up the Streams Messaging Manager database and its associated user accounts, ensuring proper configuration for Streams Messaging Manager operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(smm_username), I(smm_password), and I(smm_database) variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + smm_username: + description: The username for the Streams Messaging Manager database user and owner of the database. + type: str + required: false + default: streamsmsgmgr + smm_password: + description: The password for the Streams Messaging Manager database user. + type: str + required: false + default: streamsmsgmgr + smm_database: + description: The name of the database to be created for Streams Messaging Manager. + type: str + required: false + default: streamsmsgmgr diff --git a/roles/prereq_smm_database/molecule/default/converge.yml b/roles/prereq_smm_database/molecule/default/converge.yml new file mode 100644 index 00000000..55ffd439 --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/converge.yml @@ -0,0 +1,22 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: no + become: yes + tasks: + - name: Create Streams Messaging Manager database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_smm_database diff --git a/roles/prereq_smm_database/molecule/default/create.yml b/roles/prereq_smm_database/molecule/default/create.yml new file mode 100644 index 00000000..132438bf --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/create.yml @@ -0,0 +1,335 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: '' + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_smm_database/molecule/default/destroy.yml b/roles/prereq_smm_database/molecule/default/destroy.yml new file mode 100644 index 00000000..fb95a201 --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/destroy.yml @@ -0,0 +1,156 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: '{{ default_run_config | combine(run_config_from_file) }}' + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: '{{ platforms }}' + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_smm_database/molecule/default/molecule.yml b/roles/prereq_smm_database/molecule/default/molecule.yml new file mode 100644 index 00000000..4b162974 --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/molecule.yml @@ -0,0 +1,45 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_smm_database-rhel9-4 + Project: Molecule testing for prereq_smm_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_smm_database/molecule/default/prepare.yml b/roles/prereq_smm_database/molecule/default/prepare.yml new file mode 100644 index 00000000..2b8546dd --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: yes + become: yes + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres + \ No newline at end of file diff --git a/roles/prereq_smm_database/molecule/default/requirements.yml b/roles/prereq_smm_database/molecule/default/requirements.yml new file mode 100644 index 00000000..cd0f1849 --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql + diff --git a/roles/prereq_smm_database/molecule/default/verify.yml b/roles/prereq_smm_database/molecule/default/verify.yml new file mode 100644 index 00000000..2703afec --- /dev/null +++ b/roles/prereq_smm_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: no + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'streamsmsgmgr';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database streamsmsgmgr does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'streamsmsgmgr';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User streamsmsgmgr does not exist!" + when: user_check.query_result | length == 0 + diff --git a/roles/prereq_smm_database/tasks/main.yml b/roles/prereq_smm_database/tasks/main.yml new file mode 100644 index 00000000..21b6e01f --- /dev/null +++ b/roles/prereq_smm_database/tasks/main.yml @@ -0,0 +1,27 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ smm_details }}" + smm_details: + - user: "{{ smm_username }}" + password: "{{ smm_password }}" + db: "{{ smm_database }}" + no_log: true + + From 2445492dc327a923d5bf99d26cf5ea42dcee2b16 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:21 -0400 Subject: [PATCH 57/72] Add Apache Solr prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_solr/README.md | 53 +++ roles/prereq_solr/meta/argument_specs.yml | 23 ++ .../prereq_solr/molecule/default/converge.yml | 23 ++ roles/prereq_solr/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_solr/molecule/default/destroy.yml | 157 ++++++++ .../prereq_solr/molecule/default/molecule.yml | 49 +++ .../prereq_solr/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_solr/molecule/default/verify.yml | 36 ++ roles/prereq_solr/tasks/main.yml | 39 ++ roles/prereq_solr/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_solr/README.md create mode 100644 roles/prereq_solr/meta/argument_specs.yml create mode 100644 roles/prereq_solr/molecule/default/converge.yml create mode 100644 roles/prereq_solr/molecule/default/create.yml create mode 100644 roles/prereq_solr/molecule/default/destroy.yml create mode 100644 roles/prereq_solr/molecule/default/molecule.yml create mode 100644 roles/prereq_solr/molecule/default/prepare.yml create mode 100644 roles/prereq_solr/molecule/default/requirements.yml create mode 100644 roles/prereq_solr/molecule/default/verify.yml create mode 100644 roles/prereq_solr/tasks/main.yml create mode 100644 roles/prereq_solr/vars/main.yml diff --git a/roles/prereq_solr/README.md b/roles/prereq_solr/README.md new file mode 100644 index 00000000..fd738f17 --- /dev/null +++ b/roles/prereq_solr/README.md @@ -0,0 +1,53 @@ +# prereq_solr + +Set up for Solr + +This role prepares a host for Apache Solr usage by creating a dedicated system user and group named `solr`. This user is essential for running Solr processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Solr communication. + +The role will: +- Create the `solr` system user and group. +- Configure home directories and other necessary local paths for the `solr` user, if required. +- Ensure appropriate permissions are set for files and directories related to Solr. +- Configure TLS ACLs to secure Solr communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: solr_nodes + tasks: + - name: Set up the solr user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_solr +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_solr/meta/argument_specs.yml b/roles/prereq_solr/meta/argument_specs.yml new file mode 100644 index 00000000..2248ba4d --- /dev/null +++ b/roles/prereq_solr/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Solr + description: | + Set up for Apache Solr usage, notably, create the local C(solr) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_solr/molecule/default/converge.yml b/roles/prereq_solr/molecule/default/converge.yml new file mode 100644 index 00000000..b1d4198a --- /dev/null +++ b/roles/prereq_solr/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Solr + ansible.builtin.import_role: + name: cloudera.exe.prereq_solr diff --git a/roles/prereq_solr/molecule/default/create.yml b/roles/prereq_solr/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_solr/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_solr/molecule/default/destroy.yml b/roles/prereq_solr/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_solr/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_solr/molecule/default/molecule.yml b/roles/prereq_solr/molecule/default/molecule.yml new file mode 100644 index 00000000..8116ec83 --- /dev/null +++ b/roles/prereq_solr/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_solr-rhel9-4 + Project: Molecule testing for prereq_solr +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_solr/molecule/default/prepare.yml b/roles/prereq_solr/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_solr/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_solr/molecule/default/requirements.yml b/roles/prereq_solr/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_solr/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_solr/molecule/default/verify.yml b/roles/prereq_solr/molecule/default/verify.yml new file mode 100644 index 00000000..82dbf26a --- /dev/null +++ b/roles/prereq_solr/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for solr users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ solr_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ solr_local_accounts }}" diff --git a/roles/prereq_solr/tasks/main.yml b/roles/prereq_solr/tasks/main.yml new file mode 100644 index 00000000..e44567af --- /dev/null +++ b/roles/prereq_solr/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ solr_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ solr_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_solr/vars/main.yml b/roles/prereq_solr/vars/main.yml new file mode 100644 index 00000000..e619ce8f --- /dev/null +++ b/roles/prereq_solr/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +solr_local_accounts: + - user: solr + home: /var/lib/solr + comment: Solr + keystore_acl: true From eee52a37146ca453f1238aef98feb23423912687 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:21 -0400 Subject: [PATCH 58/72] Add Apache Spark prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_spark/README.md | 53 +++ roles/prereq_spark/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_spark/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_spark/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_spark/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_spark/molecule/default/verify.yml | 42 +++ roles/prereq_spark/tasks/main.yml | 44 +++ roles/prereq_spark/vars/main.yml | 20 ++ 11 files changed, 806 insertions(+) create mode 100644 roles/prereq_spark/README.md create mode 100644 roles/prereq_spark/meta/argument_specs.yml create mode 100644 roles/prereq_spark/molecule/default/converge.yml create mode 100644 roles/prereq_spark/molecule/default/create.yml create mode 100644 roles/prereq_spark/molecule/default/destroy.yml create mode 100644 roles/prereq_spark/molecule/default/molecule.yml create mode 100644 roles/prereq_spark/molecule/default/prepare.yml create mode 100644 roles/prereq_spark/molecule/default/requirements.yml create mode 100644 roles/prereq_spark/molecule/default/verify.yml create mode 100644 roles/prereq_spark/tasks/main.yml create mode 100644 roles/prereq_spark/vars/main.yml diff --git a/roles/prereq_spark/README.md b/roles/prereq_spark/README.md new file mode 100644 index 00000000..0197fc2b --- /dev/null +++ b/roles/prereq_spark/README.md @@ -0,0 +1,53 @@ +# prereq_spark + +Set up for Spark + +This role prepares a host for Apache Spark usage by creating a dedicated system user and group named `spark`. This user and group are essential for running Spark processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Spark communication. + +The role will: +- Create the `spark` system user and group. +- Configure home directories and other necessary local paths for the `spark` user, if required. +- Ensure appropriate permissions are set for files and directories related to Spark. +- Configure TLS ACLs to secure Spark communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: spark_nodes + tasks: + - name: Set up the spark user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_spark +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_spark/meta/argument_specs.yml b/roles/prereq_spark/meta/argument_specs.yml new file mode 100644 index 00000000..390a6b25 --- /dev/null +++ b/roles/prereq_spark/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Spark + description: | + Set up for Apache Spark usage, notably, create the local C(spark) user and local C(spark) group. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_spark/molecule/default/converge.yml b/roles/prereq_spark/molecule/default/converge.yml new file mode 100644 index 00000000..df82bf78 --- /dev/null +++ b/roles/prereq_spark/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Spark + ansible.builtin.import_role: + name: cloudera.exe.prereq_spark diff --git a/roles/prereq_spark/molecule/default/create.yml b/roles/prereq_spark/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_spark/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_spark/molecule/default/destroy.yml b/roles/prereq_spark/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_spark/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_spark/molecule/default/molecule.yml b/roles/prereq_spark/molecule/default/molecule.yml new file mode 100644 index 00000000..12c0fa18 --- /dev/null +++ b/roles/prereq_spark/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_spark-rhel9-4 + Project: Molecule testing for prereq_spark +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_spark/molecule/default/prepare.yml b/roles/prereq_spark/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_spark/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_spark/molecule/default/requirements.yml b/roles/prereq_spark/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_spark/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_spark/molecule/default/verify.yml b/roles/prereq_spark/molecule/default/verify.yml new file mode 100644 index 00000000..5eb85baa --- /dev/null +++ b/roles/prereq_spark/molecule/default/verify.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for spark group membership + ansible.builtin.command: groups spark + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("spark") + changed_when: false + + - name: Check for spark users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ spark_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ spark_local_accounts }}" diff --git a/roles/prereq_spark/tasks/main.yml b/roles/prereq_spark/tasks/main.yml new file mode 100644 index 00000000..544446f4 --- /dev/null +++ b/roles/prereq_spark/tasks/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create spark group + ansible.builtin.group: + name: spark + state: present + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ spark_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ spark_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_spark/vars/main.yml b/roles/prereq_spark/vars/main.yml new file mode 100644 index 00000000..e7dfaeb9 --- /dev/null +++ b/roles/prereq_spark/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spark_local_accounts: + - user: spark + home: /var/lib/spark + comment: Spark + keystore_acl: true From 3ca57687ce7f0c65aa5523b870c98e1825d93a16 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:22 -0400 Subject: [PATCH 59/72] Add Apache Spark2 prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_spark2/README.md | 53 +++ roles/prereq_spark2/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_spark2/molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_spark2/molecule/default/verify.yml | 32 ++ roles/prereq_spark2/tasks/main.yml | 39 ++ roles/prereq_spark2/vars/main.yml | 19 + 11 files changed, 790 insertions(+) create mode 100644 roles/prereq_spark2/README.md create mode 100644 roles/prereq_spark2/meta/argument_specs.yml create mode 100644 roles/prereq_spark2/molecule/default/converge.yml create mode 100644 roles/prereq_spark2/molecule/default/create.yml create mode 100644 roles/prereq_spark2/molecule/default/destroy.yml create mode 100644 roles/prereq_spark2/molecule/default/molecule.yml create mode 100644 roles/prereq_spark2/molecule/default/prepare.yml create mode 100644 roles/prereq_spark2/molecule/default/requirements.yml create mode 100644 roles/prereq_spark2/molecule/default/verify.yml create mode 100644 roles/prereq_spark2/tasks/main.yml create mode 100644 roles/prereq_spark2/vars/main.yml diff --git a/roles/prereq_spark2/README.md b/roles/prereq_spark2/README.md new file mode 100644 index 00000000..ec0f91f9 --- /dev/null +++ b/roles/prereq_spark2/README.md @@ -0,0 +1,53 @@ +# prereq_spark2 + +Set up for Spark2 + +This role prepares a host for Apache Spark2 usage by creating a dedicated system user and group named `spark2`. This user is essential for running Spark2 processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Spark2 communication. + +The role will: +- Create the `spark2` system user and group. +- Configure home directories and other necessary local paths for the `spark2` user, if required. +- Ensure appropriate permissions are set for files and directories related to Spark2. +- Configure TLS ACLs to secure Spark2 communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: spark2_nodes + tasks: + - name: Set up the spark2 user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_spark2 +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_spark2/meta/argument_specs.yml b/roles/prereq_spark2/meta/argument_specs.yml new file mode 100644 index 00000000..affa14d1 --- /dev/null +++ b/roles/prereq_spark2/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Spark2 + description: | + Set up for Apache Spark2 usage, notably, create the local C(spark2) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_spark2/molecule/default/converge.yml b/roles/prereq_spark2/molecule/default/converge.yml new file mode 100644 index 00000000..87c52887 --- /dev/null +++ b/roles/prereq_spark2/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Spark2 + ansible.builtin.import_role: + name: cloudera.exe.prereq_spark2 diff --git a/roles/prereq_spark2/molecule/default/create.yml b/roles/prereq_spark2/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_spark2/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_spark2/molecule/default/destroy.yml b/roles/prereq_spark2/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_spark2/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_spark2/molecule/default/molecule.yml b/roles/prereq_spark2/molecule/default/molecule.yml new file mode 100644 index 00000000..b56f87ff --- /dev/null +++ b/roles/prereq_spark2/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_spark2-rhel9-4 + Project: Molecule testing for prereq_spark2 +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_spark2/molecule/default/prepare.yml b/roles/prereq_spark2/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_spark2/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_spark2/molecule/default/requirements.yml b/roles/prereq_spark2/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_spark2/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_spark2/molecule/default/verify.yml b/roles/prereq_spark2/molecule/default/verify.yml new file mode 100644 index 00000000..3eeea186 --- /dev/null +++ b/roles/prereq_spark2/molecule/default/verify.yml @@ -0,0 +1,32 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for spark2 user + ansible.builtin.command: grep spark2 /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ spark2_local_accounts }}" diff --git a/roles/prereq_spark2/tasks/main.yml b/roles/prereq_spark2/tasks/main.yml new file mode 100644 index 00000000..a19ba7f3 --- /dev/null +++ b/roles/prereq_spark2/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ spark2_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ spark2_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_spark2/vars/main.yml b/roles/prereq_spark2/vars/main.yml new file mode 100644 index 00000000..b50d9bb1 --- /dev/null +++ b/roles/prereq_spark2/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spark2_local_accounts: + - user: spark2 + home: /var/lib/spark2 + comment: spark2 From 04573d23791d109cb5545ac6b0e88a79b8b95cd0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:23 -0400 Subject: [PATCH 60/72] Add Apache Sqoop prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_sqoop/README.md | 54 +++ roles/prereq_sqoop/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../prereq_sqoop/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_sqoop/molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../prereq_sqoop/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../prereq_sqoop/molecule/default/verify.yml | 41 +++ roles/prereq_sqoop/tasks/main.yml | 44 +++ roles/prereq_sqoop/vars/main.yml | 23 ++ 11 files changed, 809 insertions(+) create mode 100644 roles/prereq_sqoop/README.md create mode 100644 roles/prereq_sqoop/meta/argument_specs.yml create mode 100644 roles/prereq_sqoop/molecule/default/converge.yml create mode 100644 roles/prereq_sqoop/molecule/default/create.yml create mode 100644 roles/prereq_sqoop/molecule/default/destroy.yml create mode 100644 roles/prereq_sqoop/molecule/default/molecule.yml create mode 100644 roles/prereq_sqoop/molecule/default/prepare.yml create mode 100644 roles/prereq_sqoop/molecule/default/requirements.yml create mode 100644 roles/prereq_sqoop/molecule/default/verify.yml create mode 100644 roles/prereq_sqoop/tasks/main.yml create mode 100644 roles/prereq_sqoop/vars/main.yml diff --git a/roles/prereq_sqoop/README.md b/roles/prereq_sqoop/README.md new file mode 100644 index 00000000..2cb3f054 --- /dev/null +++ b/roles/prereq_sqoop/README.md @@ -0,0 +1,54 @@ +# prereq_sqoop + +Set up for Sqoop + +This role prepares a host for Apache Sqoop usage by creating dedicated system users and a group. It specifically creates the `sqoop` and `sqoop2` users, along with the `sqoop` group. These users and the group are essential for running Sqoop processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Sqoop communication. + +The role will: +- Create the `sqoop` system group. +- Create the `sqoop` and `sqoop2` system users, assigning them to the `sqoop` group. +- Configure home directories and other necessary local paths for these users, if required. +- Ensure appropriate permissions are set for files and directories related to Sqoop. +- Configure TLS ACLs to secure Sqoop communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: sqoop_nodes + tasks: + - name: Set up the sqoop users and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_sqoop +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_sqoop/meta/argument_specs.yml b/roles/prereq_sqoop/meta/argument_specs.yml new file mode 100644 index 00000000..bc700de7 --- /dev/null +++ b/roles/prereq_sqoop/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Sqoop + description: | + Set up for Apache Sqoop usage, notably, create the local C(sqoop,sqoop2) users and local C(sqoop) group. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_sqoop/molecule/default/converge.yml b/roles/prereq_sqoop/molecule/default/converge.yml new file mode 100644 index 00000000..f99bf777 --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Sqoop + ansible.builtin.import_role: + name: cloudera.exe.prereq_sqoop diff --git a/roles/prereq_sqoop/molecule/default/create.yml b/roles/prereq_sqoop/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_sqoop/molecule/default/destroy.yml b/roles/prereq_sqoop/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_sqoop/molecule/default/molecule.yml b/roles/prereq_sqoop/molecule/default/molecule.yml new file mode 100644 index 00000000..15611e41 --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_sqoop-rhel9-4 + Project: Molecule testing for prereq_sqoop +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_sqoop/molecule/default/prepare.yml b/roles/prereq_sqoop/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_sqoop/molecule/default/requirements.yml b/roles/prereq_sqoop/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_sqoop/molecule/default/verify.yml b/roles/prereq_sqoop/molecule/default/verify.yml new file mode 100644 index 00000000..073c2068 --- /dev/null +++ b/roles/prereq_sqoop/molecule/default/verify.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for sqoop-related users + ansible.builtin.command: grep {{ item }} /etc/passwd + loop: + - sqoop + - sqoop2 + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for sqoop group membership + ansible.builtin.command: groups sqoop + register: __groups + failed_when: __groups.rc != 0 and __groups.stdout is not search("sqoop") + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ sqoop_local_accounts }}" diff --git a/roles/prereq_sqoop/tasks/main.yml b/roles/prereq_sqoop/tasks/main.yml new file mode 100644 index 00000000..4a4f3207 --- /dev/null +++ b/roles/prereq_sqoop/tasks/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create sqoop group + ansible.builtin.group: + name: sqoop + state: present + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ sqoop_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ sqoop_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_sqoop/vars/main.yml b/roles/prereq_sqoop/vars/main.yml new file mode 100644 index 00000000..7165ebad --- /dev/null +++ b/roles/prereq_sqoop/vars/main.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sqoop_local_accounts: + - user: sqoop + home: /var/lib/sqoop + comment: Sqoop + - user: sqoop2 + home: /var/lib/sqoop2 + comment: Sqoop2 + extra_groups: [sqoop] From 263dfcd2451e9a87910ca4b9bb89902df8d168b0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:23 -0400 Subject: [PATCH 61/72] Add SQL Stream Builder prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_ssb/README.md | 53 +++ roles/prereq_ssb/meta/argument_specs.yml | 23 ++ .../prereq_ssb/molecule/default/converge.yml | 23 ++ roles/prereq_ssb/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_ssb/molecule/default/destroy.yml | 157 ++++++++ .../prereq_ssb/molecule/default/molecule.yml | 49 +++ roles/prereq_ssb/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_ssb/molecule/default/verify.yml | 36 ++ roles/prereq_ssb/tasks/main.yml | 39 ++ roles/prereq_ssb/vars/main.yml | 22 ++ 11 files changed, 797 insertions(+) create mode 100644 roles/prereq_ssb/README.md create mode 100644 roles/prereq_ssb/meta/argument_specs.yml create mode 100644 roles/prereq_ssb/molecule/default/converge.yml create mode 100644 roles/prereq_ssb/molecule/default/create.yml create mode 100644 roles/prereq_ssb/molecule/default/destroy.yml create mode 100644 roles/prereq_ssb/molecule/default/molecule.yml create mode 100644 roles/prereq_ssb/molecule/default/prepare.yml create mode 100644 roles/prereq_ssb/molecule/default/requirements.yml create mode 100644 roles/prereq_ssb/molecule/default/verify.yml create mode 100644 roles/prereq_ssb/tasks/main.yml create mode 100644 roles/prereq_ssb/vars/main.yml diff --git a/roles/prereq_ssb/README.md b/roles/prereq_ssb/README.md new file mode 100644 index 00000000..cb3d5e7a --- /dev/null +++ b/roles/prereq_ssb/README.md @@ -0,0 +1,53 @@ +# prereq_ssb + +Set up for SSB + +This role prepares a host for SQL Stream Builder (SSB) usage by creating a dedicated system user and group named `ssb`. This user is essential for running SSB processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure SSB communication. + +The role will: +- Create the `ssb` system user and group. +- Configure home directories and other necessary local paths for the `ssb` user, if required. +- Ensure appropriate permissions are set for files and directories related to SSB. +- Configure TLS ACLs to secure SSB communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: ssb_nodes + tasks: + - name: Set up the ssb user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_ssb +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ssb/meta/argument_specs.yml b/roles/prereq_ssb/meta/argument_specs.yml new file mode 100644 index 00000000..e6b54f95 --- /dev/null +++ b/roles/prereq_ssb/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for SSB + description: | + Set up for SQL Stream Builder (SSB) usage, notably, create the local C(ssb) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_ssb/molecule/default/converge.yml b/roles/prereq_ssb/molecule/default/converge.yml new file mode 100644 index 00000000..ae24aa69 --- /dev/null +++ b/roles/prereq_ssb/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for SSB + ansible.builtin.import_role: + name: cloudera.exe.prereq_ssb diff --git a/roles/prereq_ssb/molecule/default/create.yml b/roles/prereq_ssb/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_ssb/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ssb/molecule/default/destroy.yml b/roles/prereq_ssb/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_ssb/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ssb/molecule/default/molecule.yml b/roles/prereq_ssb/molecule/default/molecule.yml new file mode 100644 index 00000000..7d93d0bc --- /dev/null +++ b/roles/prereq_ssb/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_ssb-rhel9-4 + Project: Molecule testing for prereq_ssb +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_ssb/molecule/default/prepare.yml b/roles/prereq_ssb/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_ssb/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_ssb/molecule/default/requirements.yml b/roles/prereq_ssb/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_ssb/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_ssb/molecule/default/verify.yml b/roles/prereq_ssb/molecule/default/verify.yml new file mode 100644 index 00000000..42ce4ef9 --- /dev/null +++ b/roles/prereq_ssb/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for ssb users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ ssb_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ ssb_local_accounts }}" diff --git a/roles/prereq_ssb/tasks/main.yml b/roles/prereq_ssb/tasks/main.yml new file mode 100644 index 00000000..ef333c69 --- /dev/null +++ b/roles/prereq_ssb/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ ssb_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ ssb_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_ssb/vars/main.yml b/roles/prereq_ssb/vars/main.yml new file mode 100644 index 00000000..8be34386 --- /dev/null +++ b/roles/prereq_ssb/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ssb_local_accounts: + - user: ssb + home: /var/lib/ssb + comment: SQL Stream Builder + keystore_acl: true + key_acl: true + key_password_acl: true From be025f37a9b3ea19a4589a484c4164e1880f9bd9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:24 -0400 Subject: [PATCH 62/72] Add SQL Stream Builder database prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_ssb_database/README.md | 89 +++++ roles/prereq_ssb_database/defaults/main.yml | 28 ++ .../meta/argument_specs.yml | 75 ++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 46 +++ .../molecule/default/prepare.yml | 33 ++ .../molecule/default/requirements.yml | 22 ++ .../molecule/default/verify.yml | 49 +++ roles/prereq_ssb_database/tasks/main.yml | 28 ++ 11 files changed, 886 insertions(+) create mode 100644 roles/prereq_ssb_database/README.md create mode 100644 roles/prereq_ssb_database/defaults/main.yml create mode 100644 roles/prereq_ssb_database/meta/argument_specs.yml create mode 100644 roles/prereq_ssb_database/molecule/default/converge.yml create mode 100644 roles/prereq_ssb_database/molecule/default/create.yml create mode 100644 roles/prereq_ssb_database/molecule/default/destroy.yml create mode 100644 roles/prereq_ssb_database/molecule/default/molecule.yml create mode 100644 roles/prereq_ssb_database/molecule/default/prepare.yml create mode 100644 roles/prereq_ssb_database/molecule/default/requirements.yml create mode 100644 roles/prereq_ssb_database/molecule/default/verify.yml create mode 100644 roles/prereq_ssb_database/tasks/main.yml diff --git a/roles/prereq_ssb_database/README.md b/roles/prereq_ssb_database/README.md new file mode 100644 index 00000000..e1aea0bf --- /dev/null +++ b/roles/prereq_ssb_database/README.md @@ -0,0 +1,89 @@ +# prereq_ssb_database + +Set up database and user accounts for SQL Stream Builder + +This role automates the setup of databases and their associated user accounts for SQL Stream Builder (SSB) and its Materialized View Engine (MVE). It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates two distinct databases and dedicated users with ownership privileges for each, using sensible defaults that can be easily overridden. + +The role will: +- Connect to the specified database server using administrative credentials. +- For the SSB Admin service, it will: + - Create a new database with the name specified by `ssb_admin_database`. + - Create a new database user specified by `ssb_admin_username` with the password from `ssb_admin_password`. + - Grant ownership and all necessary privileges to the `ssb_admin_username` for the new database. +- For the SSB Materialized View Engine, it will: + - Create a new database with the name specified by `ssb_mve_database`. + - Create a new database user specified by `ssb_mve_username` with the password from `ssb_mve_password`. + - Grant ownership and all necessary privileges to the `ssb_mve_username` for the new database. + +# Requirements + +- A running and accessible database server of the specified `database_type`. +- The `database_admin_user` must have sufficient administrative privileges to create new databases and users. +- The machine running the Ansible playbook must have the necessary database client libraries installed to connect to the database (e.g., `psycopg2` for PostgreSQL, `mysql-connector-python` for MySQL). + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `database_type` | `str` | `True` | | Specifies the type of database to connect to. Valid choices are `postgresql`, `mysql`, and `oracle`. | +| `database_host` | `str` | `True` | | The hostname or IP address of the database server. | +| `database_admin_user` | `str` | `True` | | The username with administrative privileges used to manage the database. | +| `database_admin_password` | `str` | `True` | | The password for the database administrative user. This variable is marked with `no_log: true` and will not be displayed in Ansible logs. | +| `ssb_admin_username` | `str` | `False` | `ssb_admin` | The username for the SQL Stream Builder Admin database user and owner of the database. | +| `ssb_admin_password` | `str` | `False` | `ssb_admin` | The password for the SQL Stream Builder Admin database user. It is highly recommended to override this default in production. | +| `ssb_admin_database` | `str` | `False` | `ssb_admin` | The name of the database to be created for SQL Stream Builder Admin. | +| `ssb_mve_username` | `str` | `False` | `ssb_mve` | The username for the Materialized View Engine database user and owner of the database. | +| `ssb_mve_password` | `str` | `False` | `ssb_mve` | The password for the Materialized View Engine database user. It is highly recommended to override this default in production. | +| `ssb_mve_database` | `str` | `False` | `ssb_mve` | The name of the database to be created for the Materialized View Engine. | + +# Example Playbook + +```yaml +- hosts: localhost + tasks: + - name: Set up SSB databases and users on PostgreSQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_ssb_database + vars: + database_type: "postgresql" + database_host: "db-server.example.com" + database_admin_user: "postgres" + database_admin_password: "my_postgres_admin_password" # Use Ansible Vault for this + + - name: Set up SSB databases with custom credentials on MySQL + ansible.builtin.import_role: + name: cloudera.exe.prereq_ssb_database + vars: + database_type: "mysql" + database_host: "mysql-db-server.example.com" + database_admin_user: "root" + database_admin_password: "my_mysql_root_password" # Use Ansible Vault for this + ssb_admin_username: "my_ssb_admin_user" + ssb_admin_password: "a_strong_ssb_admin_password" + ssb_admin_database: "my_ssb_admin_db" + ssb_mve_username: "my_mve_user" + ssb_mve_password: "a_strong_mve_password" + ssb_mve_database: "my_mve_db" +``` + +# License + +``` +Copyright 2025 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_ssb_database/defaults/main.yml b/roles/prereq_ssb_database/defaults/main.yml new file mode 100644 index 00000000..7947556f --- /dev/null +++ b/roles/prereq_ssb_database/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +database_type: "{{ undef(hint='Please defined the database type.') }}" +database_host: "{{ undef(hint='Please defined the database server hostname or IP address.') }}" +# database_port: + +database_admin_user: "{{ undef(hint='Please defined the adminstrator username for the database server.') }}" +database_admin_password: "{{ undef(hint='Please defined the adminstrator password for the database server.') }}" + +ssb_admin_username: ssb_admin +ssb_admin_password: ssb_admin +ssb_admin_database: ssb_admin + +ssb_mve_username: ssb_mve +ssb_mve_password: ssb_mve +ssb_mve_database: ssb_mve diff --git a/roles/prereq_ssb_database/meta/argument_specs.yml b/roles/prereq_ssb_database/meta/argument_specs.yml new file mode 100644 index 00000000..874c7806 --- /dev/null +++ b/roles/prereq_ssb_database/meta/argument_specs.yml @@ -0,0 +1,75 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up database and user accounts for SQL Stream Builder + description: + - Set up the SQL Stream Builder (SSB) database and its associated user accounts, ensuring proper configuration for SSB operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(ssb_username), I(ssb_password), and + I(ssb_database) variables. + author: Cloudera Labs + options: + database_type: + description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). + type: str + required: true + choices: + - postgresql + - mysql + - oracle + database_host: + description: The hostname or IP address of the database server to establish the connection. + type: str + required: true + database_admin_user: + description: The username with administrative privileges to manage the database. + type: str + required: true + database_admin_password: + description: The password for the database administrative user. + type: str + required: true + no_log: true + ssb_admin_username: + description: The username for the SQL Stream Builder database user and owner of the database. + type: str + required: false + default: ssb_admin + ssb_admin_password: + description: The password for the SQL Stream Builder database user. + type: str + required: false + default: ssb_admin + ssb_admin_database: + description: The name of the database to be created for SQL Stream Builder. + type: str + required: false + default: ssb_admin + ssb_mve_username: + description: The username for the Materialized View Engine database user and owner of the database. + type: str + required: false + default: ssb_mve + ssb_mve_password: + description: The password for the Materialized View Engine database user. + type: str + required: false + default: ssb_mve + ssb_mve_database: + description: The name of the database to be created for Materialized View Engine. + type: str + required: false + default: ssb_mve diff --git a/roles/prereq_ssb_database/molecule/default/converge.yml b/roles/prereq_ssb_database/molecule/default/converge.yml new file mode 100644 index 00000000..ef62a9d2 --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Create SSB database and configure its associated user account. + ansible.builtin.import_role: + name: cloudera.exe.prereq_ssb_database diff --git a/roles/prereq_ssb_database/molecule/default/create.yml b/roles/prereq_ssb_database/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_ssb_database/molecule/default/destroy.yml b/roles/prereq_ssb_database/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_ssb_database/molecule/default/molecule.yml b/roles/prereq_ssb_database/molecule/default/molecule.yml new file mode 100644 index 00000000..d490a31a --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/molecule.yml @@ -0,0 +1,46 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_ssb_database-rhel9-4 + Project: Molecule testing for prereq_ssb_database +provisioner: + name: ansible + inventory: + group_vars: + all: + database_type: postgresql + database_host: localhost + database_admin_user: molecule + database_admin_password: molecule diff --git a/roles/prereq_ssb_database/molecule/default/prepare.yml b/roles/prereq_ssb_database/molecule/default/prepare.yml new file mode 100644 index 00000000..0cac8287 --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/prepare.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Setup database and configure superuser + hosts: all + gather_facts: true + become: true + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Install Postgresql server + ansible.builtin.import_role: + name: cloudera.exe.postgresql_server + + - name: Create superuser + community.postgresql.postgresql_user: + role_attr_flags: SUPERUSER + name: "{{ database_admin_user }}" + password: "{{ database_admin_password }}" + become_user: postgres diff --git a/roles/prereq_ssb_database/molecule/default/requirements.yml b/roles/prereq_ssb_database/molecule/default/requirements.yml new file mode 100644 index 00000000..a991f5d6 --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/requirements.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general + - community.postgresql diff --git a/roles/prereq_ssb_database/molecule/default/verify.yml b/roles/prereq_ssb_database/molecule/default/verify.yml new file mode 100644 index 00000000..99abb0fb --- /dev/null +++ b/roles/prereq_ssb_database/molecule/default/verify.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that users and databases are present + hosts: all + gather_facts: false + tasks: + - name: PostgreSQL + when: database_type == 'postgresql' + block: + - name: Check if the databases exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_database WHERE datname = 'ssb_admin';" + register: db_check + + - name: Fail if database does not exist + ansible.builtin.fail: + msg: "Database ssb_admin does not exist!" + when: db_check.query_result | length == 0 + + - name: Check if the users exists + community.postgresql.postgresql_query: + login_user: "{{ database_admin_user }}" + login_password: "{{ database_admin_password }}" + login_host: "{{ database_host }}" + db: "postgres" + query: "SELECT 1 FROM pg_roles WHERE rolname = 'ssb_admin';" + register: user_check + + - name: Fail if user does not exist + ansible.builtin.fail: + msg: "User ssb_admin does not exist!" + when: user_check.query_result | length == 0 diff --git a/roles/prereq_ssb_database/tasks/main.yml b/roles/prereq_ssb_database/tasks/main.yml new file mode 100644 index 00000000..86dd9b87 --- /dev/null +++ b/roles/prereq_ssb_database/tasks/main.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Provision databases and user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_database + vars: + database_accounts: "{{ ssb_details }}" + ssb_details: + - user: "{{ ssb_admin_username }}" + password: "{{ ssb_admin_password }}" + db: "{{ ssb_admin_database }}" + - user: "{{ ssb_mve_username }}" + password: "{{ ssb_mve_password }}" + db: "{{ ssb_mve_database }}" + no_log: true From 51280898fcc45a0da72aab0d2e34be29b7ade75f Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:25 -0400 Subject: [PATCH 63/72] Add Apache Superset prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_superset/README.md | 52 +++ roles/prereq_superset/meta/argument_specs.yml | 22 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 24 ++ roles/prereq_superset/tasks/main.yml | 39 ++ roles/prereq_superset/vars/main.yml | 22 ++ 10 files changed, 738 insertions(+) create mode 100644 roles/prereq_superset/README.md create mode 100644 roles/prereq_superset/meta/argument_specs.yml create mode 100644 roles/prereq_superset/molecule/default/converge.yml create mode 100644 roles/prereq_superset/molecule/default/create.yml create mode 100644 roles/prereq_superset/molecule/default/destroy.yml create mode 100644 roles/prereq_superset/molecule/default/molecule.yml create mode 100644 roles/prereq_superset/molecule/default/requirements.yml create mode 100644 roles/prereq_superset/molecule/default/verify.yml create mode 100644 roles/prereq_superset/tasks/main.yml create mode 100644 roles/prereq_superset/vars/main.yml diff --git a/roles/prereq_superset/README.md b/roles/prereq_superset/README.md new file mode 100644 index 00000000..79249f78 --- /dev/null +++ b/roles/prereq_superset/README.md @@ -0,0 +1,52 @@ +# prereq_superset + +Set up for Superset + +This role prepares a host for Apache Superset usage by creating a dedicated system user and group named `superset`. This user is essential for running Superset processes with appropriate permissions and isolation. + +The role will: +- Create the `superset` system user and group. +- Configure home directories and other necessary local paths for the `superset` user, if required. +- Ensure appropriate permissions are set for files and directories related to Superset. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: superset_nodes + tasks: + - name: Set up the superset user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_superset +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_superset/meta/argument_specs.yml b/roles/prereq_superset/meta/argument_specs.yml new file mode 100644 index 00000000..cc2eca4f --- /dev/null +++ b/roles/prereq_superset/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Superset + description: | + Set up for Apache Superset usage, notably, create the local C(superset) user. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_superset/molecule/default/converge.yml b/roles/prereq_superset/molecule/default/converge.yml new file mode 100644 index 00000000..085178a6 --- /dev/null +++ b/roles/prereq_superset/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Superset + ansible.builtin.import_role: + name: cloudera.exe.prereq_superset diff --git a/roles/prereq_superset/molecule/default/create.yml b/roles/prereq_superset/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_superset/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_superset/molecule/default/destroy.yml b/roles/prereq_superset/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_superset/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_superset/molecule/default/molecule.yml b/roles/prereq_superset/molecule/default/molecule.yml new file mode 100644 index 00000000..d2e24db4 --- /dev/null +++ b/roles/prereq_superset/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_superset-rhel9-4 + Project: Molecule testing for prereq_superset +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_superset/molecule/default/requirements.yml b/roles/prereq_superset/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_superset/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_superset/molecule/default/verify.yml b/roles/prereq_superset/molecule/default/verify.yml new file mode 100644 index 00000000..cd43558e --- /dev/null +++ b/roles/prereq_superset/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check for superset user + ansible.builtin.command: grep superset /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false diff --git a/roles/prereq_superset/tasks/main.yml b/roles/prereq_superset/tasks/main.yml new file mode 100644 index 00000000..5cd86dc6 --- /dev/null +++ b/roles/prereq_superset/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.user: + name: "{{ account.user }}" + home: "{{ account.home }}" + comment: "{{ account.comment | default(omit) }}" + groups: "{{ account.extra_groups | default(omit) }}" + append: true + uid: "{{ account.uid | default(omit) }}" + shell: "{{ account.shell | default(superset_default_shell) }}" + loop: "{{ superset_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Set local user home directory permissions + ansible.builtin.file: + path: "{{ account.home }}" + owner: "{{ account.user }}" + group: "{{ account.user }}" + mode: "{{ account.mode | default(superset_default_home_dir_mode) }}" + loop: "{{ superset_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.home }}" diff --git a/roles/prereq_superset/vars/main.yml b/roles/prereq_superset/vars/main.yml new file mode 100644 index 00000000..d678cca5 --- /dev/null +++ b/roles/prereq_superset/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +superset_local_accounts: + - user: superset + home: /var/lib/superset + comment: superset + +superset_default_shell: /sbin/nologin +superset_default_home_dir_mode: "0755" From 0b9dbf67aab12ef699701cc1f3de179c1936a10e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:26 -0400 Subject: [PATCH 64/72] Add support matrix role Signed-off-by: Webster Mudge --- roles/prereq_supported/README.md | 64 ++++ roles/prereq_supported/defaults/main.yml | 19 + .../prereq_supported/meta/argument_specs.yml | 36 ++ .../molecule/default/converge.yml | 26 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 53 +++ .../molecule/default/requirements.yml | 19 + .../molecule/unsupported_os/converge.yml | 38 ++ .../molecule/unsupported_os/molecule.yml | 57 +++ .../molecule/unsupported_os/requirements.yml | 18 + roles/prereq_supported/tasks/main.yml | 65 ++++ roles/prereq_supported/vars/RedHat-8.yml | 32 ++ roles/prereq_supported/vars/RedHat-9.yml | 40 +++ roles/prereq_supported/vars/Ubuntu-20.04.yml | 35 ++ roles/prereq_supported/vars/default.yml | 25 ++ roles/prereq_supported/vars/main.yml | 18 + 17 files changed, 1038 insertions(+) create mode 100644 roles/prereq_supported/README.md create mode 100644 roles/prereq_supported/defaults/main.yml create mode 100644 roles/prereq_supported/meta/argument_specs.yml create mode 100644 roles/prereq_supported/molecule/default/converge.yml create mode 100644 roles/prereq_supported/molecule/default/create.yml create mode 100644 roles/prereq_supported/molecule/default/destroy.yml create mode 100644 roles/prereq_supported/molecule/default/molecule.yml create mode 100644 roles/prereq_supported/molecule/default/requirements.yml create mode 100644 roles/prereq_supported/molecule/unsupported_os/converge.yml create mode 100644 roles/prereq_supported/molecule/unsupported_os/molecule.yml create mode 100644 roles/prereq_supported/molecule/unsupported_os/requirements.yml create mode 100644 roles/prereq_supported/tasks/main.yml create mode 100644 roles/prereq_supported/vars/RedHat-8.yml create mode 100644 roles/prereq_supported/vars/RedHat-9.yml create mode 100644 roles/prereq_supported/vars/Ubuntu-20.04.yml create mode 100644 roles/prereq_supported/vars/default.yml create mode 100644 roles/prereq_supported/vars/main.yml diff --git a/roles/prereq_supported/README.md b/roles/prereq_supported/README.md new file mode 100644 index 00000000..1ba4b7ab --- /dev/null +++ b/roles/prereq_supported/README.md @@ -0,0 +1,64 @@ +# prereq_supported + +Verify configuration against support matrix + +This role verifies various system and configuration settings on a target host against the official Cloudera on-premises support matrix, which is available at [supportmatrix.cloudera.com/](https://supportmatrix.cloudera.com). It is designed to be run early in a deployment pipeline to ensure that the environment meets all prerequisites before proceeding with the installation of Cloudera products. Additionally, the role defines and makes available a `support_matrix` variable that can be imported and utilized by other roles for their own specific verification needs. + +The role will: +- Collect system facts about the target host (OS version, kernel, etc.). +- Compare these facts against the requirements defined in the internal `support_matrix` data structure for the specified versions of Cloudera Manager, Cloudera Runtime, and Data Services. +- Log any discrepancies or unsupported configurations. +- The `support_matrix` variable will be available for use in subsequent tasks or roles within the same playbook. + +# Requirements + +- This role is intended to be run on the target hosts to gather accurate system facts. +- It requires a well-defined `support_matrix` data structure in its internal variables that corresponds to the official Cloudera support matrix. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cloudera_manager_version` | `str` | `True` | | The version of Cloudera Manager to validate against. | +| `cloudera_runtime_version` | `str` | `True` | | The version of Cloudera Runtime to validate against. | +| `data_services_version` | `str` | `False` | | The version of Cloudera Data Services to validate against. This is an optional parameter. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Verify host configuration against support matrix + ansible.builtin.import_role: + name: cloudera.exe.prereq_supported + vars: + cloudera_manager_version: "7.11.3" + cloudera_runtime_version: "7.1.9" + data_services_version: "1.0.0" # Optional parameter + + - name: Use the support matrix variable in a subsequent task + ansible.builtin.debug: + msg: "The supported Python version is {{ support_matrix.python_version }}" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_supported/defaults/main.yml b/roles/prereq_supported/defaults/main.yml new file mode 100644 index 00000000..5a76601a --- /dev/null +++ b/roles/prereq_supported/defaults/main.yml @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +cloudera_manager_version: "{{ undef(hint='Cloudera Manager Version') }}" +cloudera_runtime_version: "{{ undef(hint='Cloudera Runtime Version') }}" + +data_services_version: diff --git a/roles/prereq_supported/meta/argument_specs.yml b/roles/prereq_supported/meta/argument_specs.yml new file mode 100644 index 00000000..76f3c791 --- /dev/null +++ b/roles/prereq_supported/meta/argument_specs.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: "Verify configuration against support matrix" + description: + - Verification of various system and configuration settings against the Cloudera on premise support matrix available at supportmatrix.cloudera.com. + - Additionally, the I(support_matrix) variable defined in this role can be imported and used in other roles. + author: + - "Jim Enright " + options: + cloudera_manager_version: + description: Version of Cloudera Manager + type: "str" + required: true + cloudera_runtime_version: + description: Version of Cloudera Runtime + type: "str" + required: true + data_services_version: + description: Version of Cloudera Runtime + type: "str" + required: false diff --git a/roles/prereq_supported/molecule/default/converge.yml b/roles/prereq_supported/molecule/default/converge.yml new file mode 100644 index 00000000..2dbdb94d --- /dev/null +++ b/roles/prereq_supported/molecule/default/converge.yml @@ -0,0 +1,26 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Run supported prereqs role + ansible.builtin.import_role: + name: cloudera.exe.prereq_supported + vars: + cloudera_manager_version: 7.11.3 + cloudera_runtime_version: 7.1.9 diff --git a/roles/prereq_supported/molecule/default/create.yml b/roles/prereq_supported/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_supported/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_supported/molecule/default/destroy.yml b/roles/prereq_supported/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_supported/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_supported/molecule/default/molecule.yml b/roles/prereq_supported/molecule/default/molecule.yml new file mode 100644 index 00000000..866eaec0 --- /dev/null +++ b/roles/prereq_supported/molecule/default/molecule.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_supported-rhel9-4 + Project: Molecule testing for prereq_supported + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_supported-ubuntu20-04 + Project: Molecule testing for prereq_supported +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_supported/molecule/default/requirements.yml b/roles/prereq_supported/molecule/default/requirements.yml new file mode 100644 index 00000000..48f43855 --- /dev/null +++ b/roles/prereq_supported/molecule/default/requirements.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.utils diff --git a/roles/prereq_supported/molecule/unsupported_os/converge.yml b/roles/prereq_supported/molecule/unsupported_os/converge.yml new file mode 100644 index 00000000..6470d010 --- /dev/null +++ b/roles/prereq_supported/molecule/unsupported_os/converge.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: true + become: true + tasks: + - name: Run supported prereqs role on unsupported versions + block: + - name: Run role with expected failure + ansible.builtin.import_role: + name: cloudera.exe.prereq_supported + vars: + cloudera_manager_version: 7.7.3 + cloudera_runtime_version: 7.1.8 + + - name: Check execution halted + ansible.builtin.fail: + msg: "Execution should stop before this task" + register: _should_not_run + rescue: + - name: Confirm failure + ansible.builtin.assert: + that: + - _should_not_run is not defined diff --git a/roles/prereq_supported/molecule/unsupported_os/molecule.yml b/roles/prereq_supported/molecule/unsupported_os/molecule.yml new file mode 100644 index 00000000..1e89a39e --- /dev/null +++ b/roles/prereq_supported/molecule/unsupported_os/molecule.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netattr +# - boto3 + +driver: + name: ec2 +platforms: + # RHEL 9.4 + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_supported-rhel9-4 + Project: Molecule testing for prereq_supported + # Ubuntu 20.04 + - name: ubuntu20.04.molecule.internal + image_owner: "099720109477" + image_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_supported-ubuntu20-04 + Project: Molecule testing for prereq_supported +provisioner: + name: ansible + # inventory: + # group_vars: + # all: + playbooks: + create: ../default/create.yml + destroy: ../default/destroy.yml + converge: converge.yml diff --git a/roles/prereq_supported/molecule/unsupported_os/requirements.yml b/roles/prereq_supported/molecule/unsupported_os/requirements.yml new file mode 100644 index 00000000..8ac46513 --- /dev/null +++ b/roles/prereq_supported/molecule/unsupported_os/requirements.yml @@ -0,0 +1,18 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - amazon.aws + - ansible.utils diff --git a/roles/prereq_supported/tasks/main.yml b/roles/prereq_supported/tasks/main.yml new file mode 100644 index 00000000..6343c485 --- /dev/null +++ b/roles/prereq_supported/tasks/main.yml @@ -0,0 +1,65 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Load support matrix variables for OS + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +- name: Print discovered support matrix based on manager and runtime version + ansible.builtin.debug: + var: support_matrix | selectattr('manager_version', '==', (cm_version.major + '.' + cm_version.minor + '.' + cm_version.patch)) | + selectattr('runtime_version', '==', cloudera_runtime_version | regex_search('(\\d+\\.\\d+\\.\\d+)')) + verbosity: 2 + vars: + cm_version: "{{ cloudera_manager_version | cloudera.exe.cm_version }}" + +# Validation 1 - Operating Supported OS for given inputs +- name: Assert that OS is supported for Cloudera Runtime and Manager versions + ansible.builtin.assert: + that: + - support_matrix | selectattr('manager_version', '==', (cm_version.major + '.' + cm_version.minor + '.' + cm_version.patch)) | + selectattr('runtime_version', '==', cloudera_runtime_version | regex_search('(\\d+\\.\\d+\\.\\d+)')) | length > 0 + fail_msg: "OS {{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }} not supported." + vars: + cm_version: "{{ cloudera_manager_version | cloudera.exe.cm_version }}" + +# Validation 2 - Check if ECS is defined and supported +- name: Data Services validations + when: data_services_version != None + block: + - name: Asset that OS is supported for Data Services + ansible.builtin.assert: + that: + - ansible_os_family in ecs_supported_os_families + fail_msg: "ECS '{{ data_services_version }}' is not supported on OS family '{{ ansible_os_family }}'." + + - name: Assert that Data Services Version is supported for Cloudera Runtime and Manager versions + ansible.builtin.assert: + that: + - support_matrix | selectattr('manager_version', '==', (cm_version.major + '.' + cm_version.minor + '.' + cm_version.patch)) | + selectattr('runtime_version', '==', cloudera_runtime_version) | selectattr('data_services_version', 'version', data_services_version, operator='le', + version_type='semver') | length > 0 + fail_msg: >- + Data Services version '{{ data_services_version }}' not supported for runtime '{{ cloudera_runtime_version }}' and manager version '{{ cloudera_manager_version + }}'. + vars: + cm_version: "{{ cloudera_manager_version | cloudera.exe.cm_version }}" diff --git a/roles/prereq_supported/vars/RedHat-8.yml b/roles/prereq_supported/vars/RedHat-8.yml new file mode 100644 index 00000000..1d7cfaf8 --- /dev/null +++ b/roles/prereq_supported/vars/RedHat-8.yml @@ -0,0 +1,32 @@ +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Support matrix definition for given operating system family +# Format of each list item: +# manager_version: +# runtime_version: +# python_version: +# python2: # Optional, if required +# data_services_version: # Optional, if required + +# TODO: Support qualifiers for minimum versions +support_matrix: + - manager_version: "7.13.1" + runtime_version: "7.3.1" + python_version: "3.9.14" + - manager_version: "7.11.3" + runtime_version: "7.1.9" + python_version: "3.8.0" + data_services_version: "1.5.4" diff --git a/roles/prereq_supported/vars/RedHat-9.yml b/roles/prereq_supported/vars/RedHat-9.yml new file mode 100644 index 00000000..2fec417f --- /dev/null +++ b/roles/prereq_supported/vars/RedHat-9.yml @@ -0,0 +1,40 @@ +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Support matrix definition for given operating system family +# Format of each list item: +# manager_version: +# runtime_version: +# python_version: +# python2: # Optional, if required +# data_services_version: # Optional, if required + +# TODO: Support qualifiers for minimum versions +support_matrix: + - manager_version: "7.13.1" + runtime_version: "7.3.1" + python_version: "3.9.14" + - manager_version: "7.13.1" + runtime_version: "7.1.9" + python_version: "3.9.14" + data_services_version: "1.5.5" + - manager_version: "7.11.3" + runtime_version: "7.1.9" + python_version: "3.9.14" + data_services_version: "1.5.4" + - manager_version: "7.11.3" + runtime_version: "7.1.7" + python_version: "3.9.14" + python2: "2.7" diff --git a/roles/prereq_supported/vars/Ubuntu-20.04.yml b/roles/prereq_supported/vars/Ubuntu-20.04.yml new file mode 100644 index 00000000..ddffd9aa --- /dev/null +++ b/roles/prereq_supported/vars/Ubuntu-20.04.yml @@ -0,0 +1,35 @@ +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Support matrix definition for given operating system family +# Format of each list item: +# manager_version: +# runtime_version: +# python_version: +# python2: # Optional, if required + +# TODO: Support qualifiers for minimum versions + +support_matrix: + - manager_version: "7.13.1" + runtime_version: "7.3.1" + python_version: "3.8.12" + - manager_version: "7.11.3" + runtime_version: "7.1.9" + python_version: "3.8.12" + - manager_version: "7.11.3" + runtime_version: "7.1.7" + python_version: "3.8.12" + python2: "2.7" diff --git a/roles/prereq_supported/vars/default.yml b/roles/prereq_supported/vars/default.yml new file mode 100644 index 00000000..e64db561 --- /dev/null +++ b/roles/prereq_supported/vars/default.yml @@ -0,0 +1,25 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Support matrix definition for given operating system family +# Format of each list item: +# manager_version: +# runtime_version: +# python_version: +# python2: # Optional, if required +# data_services_version: # Optional, if required + +# TODO: Support qualifiers for minimum versions +support_matrix: [] diff --git a/roles/prereq_supported/vars/main.yml b/roles/prereq_supported/vars/main.yml new file mode 100644 index 00000000..0cbb421e --- /dev/null +++ b/roles/prereq_supported/vars/main.yml @@ -0,0 +1,18 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +ecs_supported_os_families: + - "RedHat" From f4ddf7d2c86a1f21909c024a709275bb64ed9cd0 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:26 -0400 Subject: [PATCH 65/72] Add Transparent Huge Pages prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_thp/README.md | 54 +++ roles/prereq_thp/handlers/main.yml | 21 ++ roles/prereq_thp/meta/argument_specs.yml | 22 ++ .../prereq_thp/molecule/default/converge.yml | 23 ++ roles/prereq_thp/molecule/default/create.yml | 336 ++++++++++++++++++ roles/prereq_thp/molecule/default/destroy.yml | 157 ++++++++ .../prereq_thp/molecule/default/molecule.yml | 42 +++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_thp/molecule/default/verify.yml | 24 ++ roles/prereq_thp/tasks/main.yml | 153 ++++++++ roles/prereq_thp/templates/cldr.conf | 20 ++ roles/prereq_thp/vars/RedHat.yml | 16 + roles/prereq_thp/vars/default.yml | 13 + roles/prereq_thp/vars/main.yml | 22 ++ 14 files changed, 924 insertions(+) create mode 100644 roles/prereq_thp/README.md create mode 100644 roles/prereq_thp/handlers/main.yml create mode 100644 roles/prereq_thp/meta/argument_specs.yml create mode 100644 roles/prereq_thp/molecule/default/converge.yml create mode 100644 roles/prereq_thp/molecule/default/create.yml create mode 100644 roles/prereq_thp/molecule/default/destroy.yml create mode 100644 roles/prereq_thp/molecule/default/molecule.yml create mode 100644 roles/prereq_thp/molecule/default/requirements.yml create mode 100644 roles/prereq_thp/molecule/default/verify.yml create mode 100644 roles/prereq_thp/tasks/main.yml create mode 100644 roles/prereq_thp/templates/cldr.conf create mode 100644 roles/prereq_thp/vars/RedHat.yml create mode 100644 roles/prereq_thp/vars/default.yml create mode 100644 roles/prereq_thp/vars/main.yml diff --git a/roles/prereq_thp/README.md b/roles/prereq_thp/README.md new file mode 100644 index 00000000..3efad016 --- /dev/null +++ b/roles/prereq_thp/README.md @@ -0,0 +1,54 @@ +# prereq_thp + +Disable Transparent Huge Pages + +This role disables Transparent Huge Pages (THP) on a host, which is a common practice for environments running big data, databases, and other performance-sensitive applications. THP can sometimes lead to performance degradation due to memory allocation overhead. The role also includes an optional step to rebuild the GRUB bootloader to ensure the THP setting persists across reboots. + +The role will: +- Modify kernel parameters to disable THP at runtime. +- Create or update a shared Cloudera profile, `/etc/tuned/cldr/tuned.conf`, if the `tuned` service is enabled. +- Modify the GRUB configuration file to add a kernel boot parameter that disables THP. +- Rebuild the GRUB bootloader configuration to apply the change permanently. +- Ensure the changes are persistent across system restarts. + +# Requirements + +- Root or `sudo` privileges are required on the target host to modify kernel parameters and GRUB configuration. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Disable Transparent Huge Pages and rebuild GRUB + ansible.builtin.import_role: + name: cloudera.exe.prereq_thp +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_thp/handlers/main.yml b/roles/prereq_thp/handlers/main.yml new file mode 100644 index 00000000..8a03ff54 --- /dev/null +++ b/roles/prereq_thp/handlers/main.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Reboot host + ansible.builtin.reboot: + +- name: Enable CLDR profile + ansible.builtin.command: tuned-adm profile cldr + changed_when: false diff --git a/roles/prereq_thp/meta/argument_specs.yml b/roles/prereq_thp/meta/argument_specs.yml new file mode 100644 index 00000000..15f0fa66 --- /dev/null +++ b/roles/prereq_thp/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Disable Transparent Huge Pages + description: | + Disable Transparent Huge Pages (THP) and, optionally, rebuild the GRUB bootloader. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_thp/molecule/default/converge.yml b/roles/prereq_thp/molecule/default/converge.yml new file mode 100644 index 00000000..20edb911 --- /dev/null +++ b/roles/prereq_thp/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Disable Transparent Huge Pages + ansible.builtin.import_role: + name: cloudera.exe.prereq_thp diff --git a/roles/prereq_thp/molecule/default/create.yml b/roles/prereq_thp/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_thp/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_thp/molecule/default/destroy.yml b/roles/prereq_thp/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_thp/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_thp/molecule/default/molecule.yml b/roles/prereq_thp/molecule/default/molecule.yml new file mode 100644 index 00000000..441578ac --- /dev/null +++ b/roles/prereq_thp/molecule/default/molecule.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_thp-rhel9-4 + Project: Molecule testing for prereq_thp +provisioner: + name: ansible + # inventory: + # group_vars: + # all: diff --git a/roles/prereq_thp/molecule/default/requirements.yml b/roles/prereq_thp/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_thp/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_thp/molecule/default/verify.yml b/roles/prereq_thp/molecule/default/verify.yml new file mode 100644 index 00000000..f793a7ef --- /dev/null +++ b/roles/prereq_thp/molecule/default/verify.yml @@ -0,0 +1,24 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Check Transparent Huge Pages + ansible.builtin.command: cat /sys/kernel/mm/transparent_hugepage/enabled + register: __thp + failed_when: __thp.stdout is not search("[never]") + changed_when: false diff --git a/roles/prereq_thp/tasks/main.yml b/roles/prereq_thp/tasks/main.yml new file mode 100644 index 00000000..952ec080 --- /dev/null +++ b/roles/prereq_thp/tasks/main.yml @@ -0,0 +1,153 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See the following: +# https://docs.cloudera.com/cdp-private-cloud-base/7.1.9/managing-clusters/topics/cm-disabling-transparent-hugepages.html +# https://stackoverflow.com/a/57779424/1629168 +# https://access.redhat.com/solutions/1320153 + +- name: Include OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['distribution'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}-{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ ansible_facts['os_family'] }}.yml" + - "default.yml" + +# TODO Review usage for non-RHEL distros +# - name: Install required packages +# ansible.builtin.package: +# name: "{{ sysfs_packages }}" +# state: latest + +# - name: Disable Transparent Huge Pages +# ansible.builtin.lineinfile: +# path: "{{ sysfs_config_file }}" +# regexp: "^kernel\/mm\/transparent\_hugepage\/enabled" +# line: kernel/mm/transparent_hugepage/enabled = never +# create: yes +# notify: reboot host + +- name: Discover GRUB configuration directory + ansible.builtin.stat: + path: /etc/default/grub.d + register: __grub_configs + +- name: Add THP fragment for GRUB configuration + when: __grub_configs.stat.exists + ansible.builtin.copy: + dest: /etc/default/grub.d/99-transparent-hugepage.cfg + content: | + GRUB_CMDLINE_LINUX_DEFAULT="$GRUB_CMDLINE_LINUX_DEFAULT transparent_hugepage=never" + mode: "0644" + register: __grub_configs_edit + +- name: Disable Transparent Huge Pages in GRUB configuration file + when: not __grub_configs.stat.exists + block: + - name: Discover GRUB configuration file + ansible.builtin.stat: + path: /etc/default/grub + register: __grub + + - name: Disable Transparent Huge Pages in GRUB configuration file + when: __grub.stat.exists + ansible.builtin.lineinfile: + path: /etc/default/grub + backup: true + backrefs: true + regexp: '^(GRUB_CMDLINE_LINUX=(?!.*hugepage)\"[^\"]+)(\".*)' + line: "\\1 transparent_hugepage=never\\2" + mode: "0644" + register: __grub_edit + +- name: Rebuild GRUB + when: __grub_configs_edit.changed or __grub_edit.changed + ansible.builtin.command: grub2-mkconfig -o /boot/grub2/grub.cfg + changed_when: true + notify: reboot host + +- name: Discover if Transparent Huge Pages are active + ansible.builtin.command: cat /sys/kernel/mm/transparent_hugepage/enabled + changed_when: false + register: __thp + +- name: Retrieve services + ansible.builtin.service_facts: + +- name: Disable Transparent Huge Pages in tuned profile + when: ('tuned.service' in ansible_facts.services) and __thp.stdout is not search("\[never\]") + block: + - name: Discover current active tuned profile + ansible.builtin.command: tuned-adm active + changed_when: false + register: __active + + - name: Set active tuned profile + when: __active.stdout is not search("cldr") + block: + - name: Create tuned profile directory for disabled THP + ansible.builtin.file: + path: /etc/tuned/cldr + state: directory + mode: "0755" + + - name: Discover existing Cloudera tuned profile + ansible.builtin.stat: + path: /etc/tuned/cldr/tuned.conf + register: __profile + + - name: Create tuned profile configuration for disabled THP + when: not __profile.stat.exists + ansible.builtin.template: + src: cldr.conf + dest: /etc/tuned/cldr/tuned.conf + mode: "0644" + vars: + previous_profile: "{{ __active.stdout.split(':').1 | trim }}" + notify: Enable CLDR profile + + - name: Update tuned profile configuration for disabled THP + when: __profile.stat.exists + community.general.ini_file: + path: /etc/tuned/cldr/tuned.conf + section: vm + option: transparent_hugepages + value: never + mode: "0644" + notify: Enable CLDR profile + +- name: Disable Transparent Huge Pages in rc.local + when: ('tuned.service' not in ansible_facts.services) and __thp.stdout is not search("\[never\]") + block: + - name: Check for rc.local + ansible.builtin.stat: + path: /etc/rc.d/rc.local + register: __rc_local + + - name: Disable Transparent Huge Pages in rc.local + when: __rc_local.stat.exists + ansible.builtin.blockinfile: + path: /etc/rc.d/rc.local + block: | + echo never > /sys/kernel/mm/transparent_hugepage/enabled + echo never > /sys/kernel/mm/transparent_hugepage/defrag + mode: "+x" + +- name: Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/roles/prereq_thp/templates/cldr.conf b/roles/prereq_thp/templates/cldr.conf new file mode 100644 index 00000000..460da677 --- /dev/null +++ b/roles/prereq_thp/templates/cldr.conf @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[main] +summary = Enable Cloudera on Premise performance +include = {{ previous_profile }} + +[vm] +transparent_hugepages = never diff --git a/roles/prereq_thp/vars/RedHat.yml b/roles/prereq_thp/vars/RedHat.yml new file mode 100644 index 00000000..c0b3f6d6 --- /dev/null +++ b/roles/prereq_thp/vars/RedHat.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +grub_executable: grub2-mkconfig diff --git a/roles/prereq_thp/vars/default.yml b/roles/prereq_thp/vars/default.yml new file mode 100644 index 00000000..c26b46e1 --- /dev/null +++ b/roles/prereq_thp/vars/default.yml @@ -0,0 +1,13 @@ +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/roles/prereq_thp/vars/main.yml b/roles/prereq_thp/vars/main.yml new file mode 100644 index 00000000..be3bf45a --- /dev/null +++ b/roles/prereq_thp/vars/main.yml @@ -0,0 +1,22 @@ +--- +# Copyright 2025 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sysfs_packages: + - sysfsutils +sysfs_config_file: /etc/sysfs.conf + +tuned_service: tuned.service + +grub_executable: grub-mkconfig From 7c6d82ad881cfac821e86c6ef98fc51f881e2e18 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:27 -0400 Subject: [PATCH 66/72] Add local TLS ACL prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_tls_acls/README.md | 98 +++++ roles/prereq_tls_acls/defaults/main.yml | 27 ++ roles/prereq_tls_acls/meta/argument_specs.yml | 136 +++++++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 55 +++ .../molecule/default/prepare.yml | 43 +++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 23 ++ roles/prereq_tls_acls/tasks/check_acls.yml | 74 ++++ roles/prereq_tls_acls/tasks/main.yml | 105 ++++++ roles/prereq_tls_acls/tasks/validate.yml | 21 ++ 13 files changed, 1119 insertions(+) create mode 100644 roles/prereq_tls_acls/README.md create mode 100644 roles/prereq_tls_acls/defaults/main.yml create mode 100644 roles/prereq_tls_acls/meta/argument_specs.yml create mode 100644 roles/prereq_tls_acls/molecule/default/converge.yml create mode 100644 roles/prereq_tls_acls/molecule/default/create.yml create mode 100644 roles/prereq_tls_acls/molecule/default/destroy.yml create mode 100644 roles/prereq_tls_acls/molecule/default/molecule.yml create mode 100644 roles/prereq_tls_acls/molecule/default/prepare.yml create mode 100644 roles/prereq_tls_acls/molecule/default/requirements.yml create mode 100644 roles/prereq_tls_acls/molecule/default/verify.yml create mode 100644 roles/prereq_tls_acls/tasks/check_acls.yml create mode 100644 roles/prereq_tls_acls/tasks/main.yml create mode 100644 roles/prereq_tls_acls/tasks/validate.yml diff --git a/roles/prereq_tls_acls/README.md b/roles/prereq_tls_acls/README.md new file mode 100644 index 00000000..8f1e17f1 --- /dev/null +++ b/roles/prereq_tls_acls/README.md @@ -0,0 +1,98 @@ +# prereq_tls_acls + +Set up local user ACLs for TLS + +This role is designed to manage and validate file system ACLs for TLS-related entities, including keystores, private keys, and key password files. It operates in two distinct modes: a `main` mode to set the ACLs and a `validate` mode to check that the ACLs are correctly applied. The role's behavior is driven by a list of users and a set of flags that define which TLS entities each user should have access to. + +Typically, TLS entity variables are set as `hostvars`. + +The role will: +- **`main` mode**: + - Iterate through the `acl_user_accounts` list. + - For each specified user, it will apply `read` file ACLs for the users' `group` to the TLS keystore, encrypted private key, unencrypted private key, and password file, as directed by the corresponding Boolean flags (`keystore_acl`, `key_acl`, etc.). + - It uses the specified TLS path variables to locate the files to which ACLs should be applied. +- **`validate` mode**: + - Iterate through the `acl_user_accounts` list. + - For each user and TLS entity, it will assert that the ACLs have been correctly set. + - This mode is useful for verification in CI/CD pipelines or after an initial deployment. + +# Requirements + +- Root or `sudo` privileges are required on the target host to set file system ACLs. +- The users listed in `acl_user_accounts` must already exist on the target host. +- The TLS files (keystore, keys, password file) must exist on the target host at the specified paths. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `acl_user_accounts` | `list` of `dict` | `False` | `[]` | A list of user accounts to which ACLs will be applied or validated. Each item is a dictionary with the following keys. | +|     `user` | `str` | `True` | | The name of the user account. | +|     `keystore_acl` | `bool` | `False` | `false` | If `true`, sets an ACL for the user on the TLS keystore. | +|     `key_acl` | `bool` | `False` | `false` | If `true`, sets an ACL for the user on the encrypted TLS private key. | +|     `key_password_acl` | `bool` | `False` | `false` | If `true`, sets an ACL for the user on the TLS private key password file. | +|     `unencrypted_key_acl` | `bool` | `False` | `false` | If `true`, sets an ACL for the user on the unencrypted TLS private key. | +| `tls_keystore_path` | `path` | `False` | | Path to the TLS keystore file. | +| `tls_keystore_path_generic` | `path` | `False` | | Path to a hardlink that points to the TLS keystore. | +| `tls_key_path` | `path` | `False` | | Path to the encrypted TLS private key file. | +| `tls_key_path_generic` | `path` | `False` | | Path to a hardlink that points to the encrypted TLS private key. | +| `tls_key_password_file` | `path` | `False` | | Path to the TLS private key password file. | +| `tls_key_path_plaintext` | `path` | `False` | | Path to the unencrypted TLS private key file. | +| `tls_key_path_plaintext_generic` | `path` | `False` | | Path to a hardlink that points to the unencrypted TLS private key. | + +# Example Playbook + +```yaml +- hosts: all + tasks: + - name: Set ACLs for Impala and Solr TLS files + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: + - user: "impala" + keystore_acl: true + key_acl: true + key_password_acl: true + - user: "solr" + keystore_acl: true + tls_keystore_path: "/opt/tls/keystore.jks" + tls_key_path: "/opt/tls/private.key" + tls_key_password_file: "/opt/tls/password.txt" + + - name: Validate TLS ACLs for Impala + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate # Use the 'validate' mode + vars: + acl_user_accounts: + - user: "impala" + keystore_acl: true + key_acl: true + key_password_acl: true + tls_keystore_path: "/opt/tls/keystore.jks" + tls_key_path: "/opt/tls/private.key" + tls_key_password_file: "/opt/tls/password.txt" +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_tls_acls/defaults/main.yml b/roles/prereq_tls_acls/defaults/main.yml new file mode 100644 index 00000000..ad5668b3 --- /dev/null +++ b/roles/prereq_tls_acls/defaults/main.yml @@ -0,0 +1,27 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO Assuming that these TLS variables are hostvars set by an upstream role +# which has created the entities and prepped the base permissions. +# +# tls_keystore_path: +# tls_keystore_path_generic: +# tls_key_path: +# tls_key_path_generic: +# tls_key_password_file: +# tls_key_path_plaintext: +# tls_key_path_plaintext_generic: + +acl_user_accounts: [] diff --git a/roles/prereq_tls_acls/meta/argument_specs.yml b/roles/prereq_tls_acls/meta/argument_specs.yml new file mode 100644 index 00000000..aebc533d --- /dev/null +++ b/roles/prereq_tls_acls/meta/argument_specs.yml @@ -0,0 +1,136 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up local user ACLs for TLS + description: | + Set up local user ACLs for TLS entities, i.e. TLS keystore, private key, and password file. + The TLS entity variables are typically set as C(hostvars). + author: Cloudera Labs + options: + acl_user_accounts: + description: A list of user accounts to apply to the TLS entities. + type: list + elements: dict + default: [] + options: + user: + description: User account name + required: true + keystore_acl: + description: Flag to set ACL on TLS keystore variations. + type: bool + default: false + key_acl: + description: Flag to set ACL on TLS private key variations. + type: bool + default: false + key_password_acl: + description: Flag to set ACL on TLS private key password file variations. + type: bool + default: false + unencrypted_key_acl: + description: Flag to set ACL on unencrypted TLS private key variations. + type: bool + default: false + tls_keystore_path: + description: + - Path of the TLS keystore. + type: path + tls_keystore_path_generic: + description: + - Path of the hardlink to the TLS keystore. + type: path + tls_key_path: + description: + - Path of the encrypted TLS private key. + type: path + tls_key_path_generic: + description: + - Path of the hardlink to the encrypted TLS private key. + type: path + tls_key_password_file: + description: + - Path of the TLS private key password file. + type: path + tls_key_path_plaintext: + description: + - Path of the unencrypted TLS private key. + type: path + tls_key_path_plaintext_generic: + description: + - Path of the hardlink to the unencrypted TLS private key. + type: path + validate: + short_description: Validate local user ACLs for TLS + description: | + Assert validity of local user ACLs for TLS entities, i.e. TLS keystore, private key, and password file. + The TLS entity variables are typically set as C(hostvars). + author: Cloudera Labs + options: + acl_user_accounts: + description: A list of user accounts to check for TLS entity ACLs. + type: list + elements: dict + default: [] + options: + user: + description: User account name + required: true + keystore_acl: + description: Flag to set ACL on TLS keystore variations. + type: bool + default: false + key_acl: + description: Flag to set ACL on TLS private key variations. + type: bool + default: false + key_password_acl: + description: Flag to set ACL on TLS private key password file variations. + type: bool + default: false + unencrypted_key_acl: + description: Flag to set ACL on unencrypted TLS private key variations. + type: bool + default: false + tls_keystore_path: + description: + - Path of the TLS keystore. + type: path + tls_keystore_path_generic: + description: + - Path of the hardlink to the TLS keystore. + type: path + tls_key_path: + description: + - Path of the encrypted TLS private key. + type: path + tls_key_path_generic: + description: + - Path of the hardlink to the encrypted TLS private key. + type: path + tls_key_password_file: + description: + - Path of the TLS private key password file. + type: path + tls_key_path_plaintext: + description: + - Path of the unencrypted TLS private key. + type: path + tls_key_path_plaintext_generic: + description: + - Path of the hardlink to the unencrypted TLS private key. + type: path diff --git a/roles/prereq_tls_acls/molecule/default/converge.yml b/roles/prereq_tls_acls/molecule/default/converge.yml new file mode 100644 index 00000000..830145e7 --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up local user ACLs + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls diff --git a/roles/prereq_tls_acls/molecule/default/create.yml b/roles/prereq_tls_acls/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_tls_acls/molecule/default/destroy.yml b/roles/prereq_tls_acls/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_tls_acls/molecule/default/molecule.yml b/roles/prereq_tls_acls/molecule/default/molecule.yml new file mode 100644 index 00000000..fc7d6cdc --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/molecule.yml @@ -0,0 +1,55 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-cloudera.exe.prereq_tls_acls-rhel9-4 + Project: Molecule testing for cloudera.exe.prereq_tls_acls +provisioner: + name: ansible + inventory: + group_vars: + all: + acl_user_accounts: + - user: test + keystore_acl: true + # key_acl: true + # key_password_acl: true + unencrypted_key_acl: true + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + # tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + # tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_tls_acls/molecule/default/prepare.yml b/roles/prereq_tls_acls/molecule/default/prepare.yml new file mode 100644 index 00000000..ac692171 --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/prepare.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create test user account + ansible.builtin.user: + name: "test" + comment: "Molecule test user" + + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_tls_acls/molecule/default/requirements.yml b/roles/prereq_tls_acls/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_tls_acls/molecule/default/verify.yml b/roles/prereq_tls_acls/molecule/default/verify.yml new file mode 100644 index 00000000..8a4dd957 --- /dev/null +++ b/roles/prereq_tls_acls/molecule/default/verify.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: Validate TLS ACLs + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml diff --git a/roles/prereq_tls_acls/tasks/check_acls.yml b/roles/prereq_tls_acls/tasks/check_acls.yml new file mode 100644 index 00000000..4efcb218 --- /dev/null +++ b/roles/prereq_tls_acls/tasks/check_acls.yml @@ -0,0 +1,74 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Variable 'account' set in outer loop + +- name: Check modified TLS ACLs + when: lookup('vars', item.file, default=False) and item.acl in account + ansible.builtin.command: "getfacl {{ lookup('vars', item.file) }}" + loop: + - file: tls_keystore_path + acl: keystore_acl + scope: group + - file: tls_keystore_path_generic + acl: keystore_acl + scope: group + - file: tls_key_path + acl: key_acl + scope: group + - file: tls_key_path_generic + acl: key_acl + scope: group + - file: tls_key_password_file + acl: key_password_acl + scope: user + - file: tls_key_path_plaintext + acl: unencrypted_key_acl + scope: group + - file: tls_key_path_plaintext_generic + acl: unencrypted_key_acl + scope: group + register: __acl_set + failed_when: __acl_set.stdout is not search(item.scope + ":" + account.user + ":r--") + changed_when: false + +- name: Check unmodified TLS ACLs + when: lookup('vars', item.file, default=False) and item.acl not in account + ansible.builtin.command: "getfacl {{ lookup('vars', item.file) }}" + loop: + - file: tls_keystore_path + acl: keystore_acl + scope: group + - file: tls_keystore_path_generic + acl: keystore_acl + scope: group + - file: tls_key_path + acl: key_acl + scope: group + - file: tls_key_path_generic + acl: key_acl + scope: group + - file: tls_key_password_file + acl: key_password_acl + scope: user + - file: tls_key_path_plaintext + acl: unencrypted_key_acl + scope: group + - file: tls_key_path_plaintext_generic + acl: unencrypted_key_acl + scope: group + register: __acl_unset + failed_when: __acl_unset.stdout is search(item.scope + ":" + account.user + ":r--") + changed_when: false diff --git a/roles/prereq_tls_acls/tasks/main.yml b/roles/prereq_tls_acls/tasks/main.yml new file mode 100644 index 00000000..c096b0cd --- /dev/null +++ b/roles/prereq_tls_acls/tasks/main.yml @@ -0,0 +1,105 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Add local user ACLs to TLS keystore + when: tls_keystore_path is defined + ansible.posix.acl: + path: "{{ tls_keystore_path }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('keystore_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS keystore hard link + when: tls_keystore_path_generic is defined + ansible.posix.acl: + path: "{{ tls_keystore_path_generic }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('keystore_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS private key + when: tls_key_path is defined + ansible.posix.acl: + path: "{{ tls_key_path }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('key_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS private key hard link + when: tls_key_path_generic is defined + ansible.posix.acl: + path: "{{ tls_key_path_generic }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('key_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS private key password file + when: tls_key_password_file is defined + ansible.posix.acl: + path: "{{ tls_key_password_file }}" + entity: "{{ account.user }}" + etype: user + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('key_password_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS private unencrypted key + when: tls_key_path_plaintext is defined + ansible.posix.acl: + path: "{{ tls_key_path_plaintext }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('unencrypted_key_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + +- name: Add local user ACLs to TLS private unencrypted key hard link + when: tls_key_path_plaintext_generic is defined + ansible.posix.acl: + path: "{{ tls_key_path_plaintext_generic }}" + entity: "{{ account.user }}" + etype: group + permissions: r + state: present + loop: "{{ acl_user_accounts | rejectattr('unencrypted_key_acl', 'undefined') | list }}" + loop_control: + loop_var: account + label: "{{ account.user }}" diff --git a/roles/prereq_tls_acls/tasks/validate.yml b/roles/prereq_tls_acls/tasks/validate.yml new file mode 100644 index 00000000..41659797 --- /dev/null +++ b/roles/prereq_tls_acls/tasks/validate.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Validate local user TLS entity ACLs + ansible.builtin.include_tasks: check_acls.yml + loop: "{{ acl_user_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" From e1a072d1d952a9140b3893d875dbae37b198d332 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:28 -0400 Subject: [PATCH 67/72] Add Apache Hadoop YARN prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_yarn/README.md | 53 +++ roles/prereq_yarn/meta/argument_specs.yml | 23 ++ .../prereq_yarn/molecule/default/converge.yml | 23 ++ roles/prereq_yarn/molecule/default/create.yml | 336 ++++++++++++++++++ .../prereq_yarn/molecule/default/destroy.yml | 157 ++++++++ .../prereq_yarn/molecule/default/molecule.yml | 49 +++ .../prereq_yarn/molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ roles/prereq_yarn/molecule/default/verify.yml | 41 +++ roles/prereq_yarn/tasks/main.yml | 39 ++ roles/prereq_yarn/vars/main.yml | 20 ++ 11 files changed, 800 insertions(+) create mode 100644 roles/prereq_yarn/README.md create mode 100644 roles/prereq_yarn/meta/argument_specs.yml create mode 100644 roles/prereq_yarn/molecule/default/converge.yml create mode 100644 roles/prereq_yarn/molecule/default/create.yml create mode 100644 roles/prereq_yarn/molecule/default/destroy.yml create mode 100644 roles/prereq_yarn/molecule/default/molecule.yml create mode 100644 roles/prereq_yarn/molecule/default/prepare.yml create mode 100644 roles/prereq_yarn/molecule/default/requirements.yml create mode 100644 roles/prereq_yarn/molecule/default/verify.yml create mode 100644 roles/prereq_yarn/tasks/main.yml create mode 100644 roles/prereq_yarn/vars/main.yml diff --git a/roles/prereq_yarn/README.md b/roles/prereq_yarn/README.md new file mode 100644 index 00000000..78448ea4 --- /dev/null +++ b/roles/prereq_yarn/README.md @@ -0,0 +1,53 @@ +# prereq_yarn + +Set up for YARN + +This role prepares a host for Apache Hadoop YARN usage by creating a dedicated system user and group named `yarn`. This user is essential for running YARN processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure YARN communication. + +The role will: +- Create the `yarn` system user and group. +- Configure home directories and other necessary local paths for the `yarn` user, if required. +- Ensure appropriate permissions are set for files and directories related to YARN. +- Configure TLS ACLs to secure YARN communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: yarn_nodes + tasks: + - name: Set up the yarn user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_yarn +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_yarn/meta/argument_specs.yml b/roles/prereq_yarn/meta/argument_specs.yml new file mode 100644 index 00000000..e6cef6c6 --- /dev/null +++ b/roles/prereq_yarn/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for YARN + description: | + Set up for Apache Hadoop YARN usage, notably, create the local C(yarn) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_yarn/molecule/default/converge.yml b/roles/prereq_yarn/molecule/default/converge.yml new file mode 100644 index 00000000..8be868cf --- /dev/null +++ b/roles/prereq_yarn/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Yarn + ansible.builtin.import_role: + name: cloudera.exe.prereq_yarn diff --git a/roles/prereq_yarn/molecule/default/create.yml b/roles/prereq_yarn/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_yarn/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_yarn/molecule/default/destroy.yml b/roles/prereq_yarn/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_yarn/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_yarn/molecule/default/molecule.yml b/roles/prereq_yarn/molecule/default/molecule.yml new file mode 100644 index 00000000..2ead28dc --- /dev/null +++ b/roles/prereq_yarn/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_yarn-rhel9-4 + Project: Molecule testing for prereq_yarn +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_yarn/molecule/default/prepare.yml b/roles/prereq_yarn/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_yarn/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_yarn/molecule/default/requirements.yml b/roles/prereq_yarn/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_yarn/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_yarn/molecule/default/verify.yml b/roles/prereq_yarn/molecule/default/verify.yml new file mode 100644 index 00000000..31bc000b --- /dev/null +++ b/roles/prereq_yarn/molecule/default/verify.yml @@ -0,0 +1,41 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for hadoop and spark group + ansible.builtin.command: grep {{ item }} /etc/group + loop: + - hadoop + - spark + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check for yarn user + ansible.builtin.command: grep yarn /etc/passwd + register: result + failed_when: result.rc != 0 + changed_when: false + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ yarn_local_accounts }}" diff --git a/roles/prereq_yarn/tasks/main.yml b/roles/prereq_yarn/tasks/main.yml new file mode 100644 index 00000000..06519677 --- /dev/null +++ b/roles/prereq_yarn/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ yarn_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ yarn_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_yarn/vars/main.yml b/roles/prereq_yarn/vars/main.yml new file mode 100644 index 00000000..63fe1782 --- /dev/null +++ b/roles/prereq_yarn/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn_local_accounts: + - user: yarn + home: /var/lib/hadoop-yarn + comment: Hadoop Yarn + extra_groups: [hadoop, spark] From 91022175998041f49bef5302980cb5c19ddfdbc9 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:28 -0400 Subject: [PATCH 68/72] Add Apache Zeppelin prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_zeppelin/README.md | 53 +++ roles/prereq_zeppelin/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_zeppelin/tasks/main.yml | 39 ++ roles/prereq_zeppelin/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_zeppelin/README.md create mode 100644 roles/prereq_zeppelin/meta/argument_specs.yml create mode 100644 roles/prereq_zeppelin/molecule/default/converge.yml create mode 100644 roles/prereq_zeppelin/molecule/default/create.yml create mode 100644 roles/prereq_zeppelin/molecule/default/destroy.yml create mode 100644 roles/prereq_zeppelin/molecule/default/molecule.yml create mode 100644 roles/prereq_zeppelin/molecule/default/prepare.yml create mode 100644 roles/prereq_zeppelin/molecule/default/requirements.yml create mode 100644 roles/prereq_zeppelin/molecule/default/verify.yml create mode 100644 roles/prereq_zeppelin/tasks/main.yml create mode 100644 roles/prereq_zeppelin/vars/main.yml diff --git a/roles/prereq_zeppelin/README.md b/roles/prereq_zeppelin/README.md new file mode 100644 index 00000000..1c58c87f --- /dev/null +++ b/roles/prereq_zeppelin/README.md @@ -0,0 +1,53 @@ +# prereq_zeppelin + +Set up for Zeppelin + +This role prepares a host for Apache Zeppelin usage by creating a dedicated system user and group named `zeppelin`. This user is essential for running Zeppelin processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure Zeppelin communication. + +The role will: +- Create the `zeppelin` system user and group. +- Configure home directories and other necessary local paths for the `zeppelin` user, if required. +- Ensure appropriate permissions are set for files and directories related to Zeppelin. +- Configure TLS ACLs to secure Zeppelin communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: zeppelin_nodes + tasks: + - name: Set up the zeppelin user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_zeppelin +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_zeppelin/meta/argument_specs.yml b/roles/prereq_zeppelin/meta/argument_specs.yml new file mode 100644 index 00000000..a6f44c34 --- /dev/null +++ b/roles/prereq_zeppelin/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Zeppelin + description: | + Set up for Apache Zeppelin usage, notably, create the local C(zeppelin) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_zeppelin/molecule/default/converge.yml b/roles/prereq_zeppelin/molecule/default/converge.yml new file mode 100644 index 00000000..24a0b2ad --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Zeppelin + ansible.builtin.import_role: + name: cloudera.exe.prereq_zeppelin diff --git a/roles/prereq_zeppelin/molecule/default/create.yml b/roles/prereq_zeppelin/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_zeppelin/molecule/default/destroy.yml b/roles/prereq_zeppelin/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_zeppelin/molecule/default/molecule.yml b/roles/prereq_zeppelin/molecule/default/molecule.yml new file mode 100644 index 00000000..f1bdbba4 --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_zeppelin-rhel9-4 + Project: Molecule testing for prereq_zeppelin +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_zeppelin/molecule/default/prepare.yml b/roles/prereq_zeppelin/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_zeppelin/molecule/default/requirements.yml b/roles/prereq_zeppelin/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_zeppelin/molecule/default/verify.yml b/roles/prereq_zeppelin/molecule/default/verify.yml new file mode 100644 index 00000000..37d039a3 --- /dev/null +++ b/roles/prereq_zeppelin/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for zeppelin users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ zeppelin_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ zeppelin_local_accounts }}" diff --git a/roles/prereq_zeppelin/tasks/main.yml b/roles/prereq_zeppelin/tasks/main.yml new file mode 100644 index 00000000..cf8cdaba --- /dev/null +++ b/roles/prereq_zeppelin/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ zeppelin_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ zeppelin_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_zeppelin/vars/main.yml b/roles/prereq_zeppelin/vars/main.yml new file mode 100644 index 00000000..5d61485f --- /dev/null +++ b/roles/prereq_zeppelin/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +zeppelin_local_accounts: + - user: zeppelin + home: /var/lib/zeppelin + comment: Zeppelin + keystore_acl: true From 1757e1305bba521fa189a83bd6d13002a1e63eb6 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:29 -0400 Subject: [PATCH 69/72] Add Apache ZooKeeper prerequisites role Signed-off-by: Webster Mudge --- roles/prereq_zookeeper/README.md | 53 +++ .../prereq_zookeeper/meta/argument_specs.yml | 23 ++ .../molecule/default/converge.yml | 23 ++ .../molecule/default/create.yml | 336 ++++++++++++++++++ .../molecule/default/destroy.yml | 157 ++++++++ .../molecule/default/molecule.yml | 49 +++ .../molecule/default/prepare.yml | 38 ++ .../molecule/default/requirements.yml | 21 ++ .../molecule/default/verify.yml | 36 ++ roles/prereq_zookeeper/tasks/main.yml | 39 ++ roles/prereq_zookeeper/vars/main.yml | 20 ++ 11 files changed, 795 insertions(+) create mode 100644 roles/prereq_zookeeper/README.md create mode 100644 roles/prereq_zookeeper/meta/argument_specs.yml create mode 100644 roles/prereq_zookeeper/molecule/default/converge.yml create mode 100644 roles/prereq_zookeeper/molecule/default/create.yml create mode 100644 roles/prereq_zookeeper/molecule/default/destroy.yml create mode 100644 roles/prereq_zookeeper/molecule/default/molecule.yml create mode 100644 roles/prereq_zookeeper/molecule/default/prepare.yml create mode 100644 roles/prereq_zookeeper/molecule/default/requirements.yml create mode 100644 roles/prereq_zookeeper/molecule/default/verify.yml create mode 100644 roles/prereq_zookeeper/tasks/main.yml create mode 100644 roles/prereq_zookeeper/vars/main.yml diff --git a/roles/prereq_zookeeper/README.md b/roles/prereq_zookeeper/README.md new file mode 100644 index 00000000..5991cc7e --- /dev/null +++ b/roles/prereq_zookeeper/README.md @@ -0,0 +1,53 @@ +# prereq_zookeeper + +Set up for Zookeeper + +This role prepares a host for Apache ZooKeeper usage by creating a dedicated system user and group named `zookeeper`. This user is essential for running ZooKeeper processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure ZooKeeper communication. + +The role will: +- Create the `zookeeper` system user and group. +- Configure home directories and other necessary local paths for the `zookeeper` user, if required. +- Ensure appropriate permissions are set for files and directories related to ZooKeeper. +- Configure TLS ACLs to secure ZooKeeper communication, if needed. + +# Requirements + +- Root or `sudo` privileges are required on the target host to create system users and groups, and to configure file system permissions and ACLs. + +# Dependencies + +None. + +# Parameters + +| Variable | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| | | | | This role has no configurable parameters. | + +# Example Playbook + +```yaml +- hosts: zookeeper_nodes + tasks: + - name: Set up the zookeeper user and environment + ansible.builtin.import_role: + name: cloudera.exe.prereq_zookeeper +``` + +# License + +``` +Copyright 2024 Cloudera, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/roles/prereq_zookeeper/meta/argument_specs.yml b/roles/prereq_zookeeper/meta/argument_specs.yml new file mode 100644 index 00000000..5f1ccbe4 --- /dev/null +++ b/roles/prereq_zookeeper/meta/argument_specs.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +argument_specs: + main: + short_description: Set up for Zookeeper + description: | + Set up for Apache Zookeeper usage, notably, create the local C(zookeeper) user. + Optionally, set up ACLs on TLS entities. + author: Cloudera Labs + options: {} diff --git a/roles/prereq_zookeeper/molecule/default/converge.yml b/roles/prereq_zookeeper/molecule/default/converge.yml new file mode 100644 index 00000000..df55b245 --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/converge.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Converge + hosts: all + gather_facts: false + become: true + tasks: + - name: Set up for Zookeeper + ansible.builtin.import_role: + name: cloudera.exe.prereq_zookeeper diff --git a/roles/prereq_zookeeper/molecule/default/create.yml b/roles/prereq_zookeeper/molecule/default/create.yml new file mode 100644 index 00000000..41583e5e --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/create.yml @@ -0,0 +1,336 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_assign_public_ip: true + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_boot_wait_seconds: 120 + default_instance_type: t3a.medium + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_private_key_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/id_rsa" + default_public_key_path: "{{ default_private_key_path }}.pub" + default_ssh_user: ansible + default_ssh_port: 22 + default_user_data: "" + + default_security_group_name: "molecule-{{ run_config.run_id }}" + default_security_group_description: Ephemeral security group for Molecule instances + default_security_group_rules: + - proto: tcp + from_port: "{{ default_ssh_port }}" + to_port: "{{ default_ssh_port }}" + cidr_ip: "0.0.0.0/0" + - proto: icmp + from_port: 8 + to_port: -1 + cidr_ip: "0.0.0.0/0" + default_security_group_rules_egress: + - proto: -1 + from_port: 0 + to_port: 0 + cidr_ip: "0.0.0.0/0" + + platform_defaults: + assign_public_ip: "{{ default_assign_public_ip }}" + aws_profile: "{{ default_aws_profile }}" + boot_wait_seconds: "{{ default_boot_wait_seconds }}" + instance_type: "{{ default_instance_type }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + private_key_path: "{{ default_private_key_path }}" + public_key_path: "{{ default_public_key_path }}" + security_group_name: "{{ default_security_group_name }}" + security_group_description: "{{ default_security_group_description }}" + security_group_rules: "{{ default_security_group_rules }}" + security_group_rules_egress: "{{ default_security_group_rules_egress }}" + ssh_user: "{{ default_ssh_user }}" + ssh_port: "{{ default_ssh_port }}" + cloud_config: {} + image: "" + image_name: "" + image_owner: [self] + name: "" + region: "" + security_groups: [] + tags: {} + volumes: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.assign_public_ip is boolean + - platform.aws_profile is string + - platform.boot_wait_seconds is integer and platform.boot_wait_seconds >= 0 + - platform.cloud_config is mapping + - platform.image is string + - platform.image_name is string + - platform.image_owner is sequence or (platform.image_owner is string and platform.image_owner | length > 0) + - platform.instance_type is string and platform.instance_type | length > 0 + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.private_key_path is string and platform.private_key_path | length > 0 + - platform.public_key_path is string and platform.public_key_path | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_group_description is string and platform.security_group_description | length > 0 + - platform.security_group_rules is sequence + - platform.security_group_rules_egress is sequence + - platform.security_groups is sequence + - platform.ssh_user is string and platform.ssh_user | length > 0 + - platform.ssh_port is integer and platform.ssh_port in range(1, 65536) + - platform.tags is mapping + - platform.volumes is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Write run config to file + ansible.builtin.copy: + dest: "{{ run_config_path }}" + content: "{{ run_config | to_yaml }}" + mode: "0600" + + - name: Generate local key pairs + community.crypto.openssh_keypair: + path: "{{ item.private_key_path }}" + type: rsa + size: 2048 + regenerate: never + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: local_keypairs + + - name: Look up EC2 AMI(s) by owner and name (if image not set) + amazon.aws.ec2_ami_info: + owners: "{{ item.image_owner }}" + filters: "{{ item.image_filters | default({}) | combine(image_name_map) }}" + vars: + image_name_map: "{% if item.image_name is defined and item.image_name | length > 0 %}{{ {'name': item.image_name} }}{% else %}{}{% endif %}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.image + register: ami_info + + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: + - platform.image or (ami_info.results[index].images | length > 0) + - platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Create ephemeral EC2 keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + key_material: "{{ local_keypair.public_key }}" + vars: + local_keypair: "{{ local_keypairs.results[index] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + register: ec2_keys + + - name: Create ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + iam_instance_profile: "{{ item.iam_instance_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + description: "{{ item.security_group_description }}" + rules: "{{ item.security_group_rules }}" + rules_egress: "{{ item.security_group_rules_egress }}" + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Create ephemeral EC2 instance(s) + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + filters: "{{ platform_filters }}" + instance_type: "{{ item.instance_type }}" + image_id: "{{ platform_image_id }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + security_groups: "{{ platform_security_groups }}" + network: + assign_public_ip: "{{ item.assign_public_ip }}" + volumes: "{{ item.volumes }}" + key_name: "{{ (item.key_inject_method == 'ec2') | ternary(item.key_name, omit) }}" + tags: "{{ platform_tags }}" + user_data: "{{ platform_user_data }}" + state: "running" + wait: true + vars: + platform_security_groups: "{{ item.security_groups or [item.security_group_name] }}" + platform_generated_image_id: "{{ (ami_info.results[index].images | sort(attribute='creation_date', reverse=True))[0].image_id }}" + platform_image_id: "{{ item.image or platform_generated_image_id }}" + + platform_generated_cloud_config: + users: + - name: "{{ item.ssh_user }}" + ssh_authorized_keys: + - "{{ local_keypairs.results[index].public_key }}" + sudo: "ALL=(ALL) NOPASSWD:ALL" + platform_cloud_config: >- + {{ (item.key_inject_method == 'cloud-init') + | ternary((item.cloud_config | combine(platform_generated_cloud_config)), item.cloud_config) }} + platform_user_data: |- + #cloud-config + {{ platform_cloud_config | to_yaml }} + + platform_generated_tags: + instance: "{{ item.name }}" + "molecule-run-id": "{{ run_config.run_id }}" + platform_tags: "{{ (item.tags or {}) | combine(platform_generated_tags) }}" + platform_filter_keys: "{{ platform_generated_tags.keys() | map('regex_replace', '^(.+)$', 'tag:\\1') }}" + platform_filters: "{{ dict(platform_filter_keys | zip(platform_generated_tags.values())) }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Instance boot block + when: ec2_instances_async is changed + block: + - name: Wait for instance creation to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Collect instance configs + ansible.builtin.set_fact: + instance_config: + instance: "{{ item.name }}" + address: "{{ item.assign_public_ip | ternary(instance.public_ip_address, instance.private_ip_address) }}" + user: "{{ item.ssh_user }}" + port: "{{ item.ssh_port }}" + identity_file: "{{ item.private_key_path }}" + instance_ids: + - "{{ instance.instance_id }}" + vars: + instance: "{{ ec2_instances.results[index].instances[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + register: instance_configs + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: >- + {{ instance_configs.results + | map(attribute='ansible_facts.instance_config') + | list + | to_json + | from_json + | to_yaml }} + mode: "0600" + + - name: Start SSH pollers + ansible.builtin.wait_for: + host: "{{ item.address }}" + port: "{{ item.port }}" + search_regex: SSH + delay: 10 + timeout: 320 + loop: "{{ instance_configs.results | map(attribute='ansible_facts.instance_config') | list }}" + loop_control: + label: "{{ item.instance }}" + register: ssh_wait_async + async: 300 + poll: 0 + + - name: Wait for SSH + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ssh_wait_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ssh_wait + until: ssh_wait is finished + retries: 300 + delay: 1 + + - name: Wait for boot process to finish + ansible.builtin.pause: + seconds: "{{ platforms | map(attribute='boot_wait_seconds') | max }}" diff --git a/roles/prereq_zookeeper/molecule/default/destroy.yml b/roles/prereq_zookeeper/molecule/default/destroy.yml new file mode 100644 index 00000000..a090fe0b --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/destroy.yml @@ -0,0 +1,157 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Destroy + hosts: localhost + connection: local + gather_facts: false + no_log: "{{ molecule_no_log }}" + vars: + # Run config handling + default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}" + default_run_config: + run_id: "{{ default_run_id }}" + + run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml" + run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}" + run_config: "{{ default_run_config | combine(run_config_from_file) }}" + + # Platform settings handling + default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}" + default_key_inject_method: cloud-init # valid values: [cloud-init, ec2] + default_key_name: "molecule-{{ run_config.run_id }}" + default_security_group_name: "molecule-{{ run_config.run_id }}" + + platform_defaults: + aws_profile: "{{ default_aws_profile }}" + key_inject_method: "{{ default_key_inject_method }}" + key_name: "{{ default_key_name }}" + region: "" + security_group_name: "{{ default_security_group_name }}" + security_groups: [] + vpc_id: "" + vpc_subnet_id: "" + + # Merging defaults into a list of dicts is, it turns out, not straightforward + platforms: >- + {{ [platform_defaults | dict2items] + | product(molecule_yml.platforms | map('dict2items') | list) + | map('flatten', levels=1) + | list + | map('items2dict') + | list }} + + # Stored instance config + instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}" + pre_tasks: + - name: Validate platform configurations + ansible.builtin.assert: + that: + - platforms | length > 0 + - platform.name is string and platform.name | length > 0 + - platform.aws_profile is string + - platform.key_inject_method is in ["cloud-init", "ec2"] + - platform.key_name is string and platform.key_name | length > 0 + - platform.region is string + - platform.security_group_name is string and platform.security_group_name | length > 0 + - platform.security_groups is sequence + - platform.vpc_id is string + - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0 + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + label: "{{ platform.name }}" + tasks: + - name: Look up subnets to determine VPCs (if needed) + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + subnet_ids: "{{ item.vpc_subnet_id }}" + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + when: not item.vpc_id + register: subnet_info + + - name: Validate discovered information + ansible.builtin.assert: + that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0) + quiet: true + loop: "{{ platforms }}" + loop_control: + loop_var: platform + index_var: index + label: "{{ platform.name }}" + + - name: Destroy resources + when: instance_config | length != 0 + block: + - name: Destroy ephemeral EC2 instances + amazon.aws.ec2_instance: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}" + vpc_subnet_id: "{{ item.vpc_subnet_id }}" + state: absent + loop: "{{ platforms }}" + loop_control: + label: "{{ item.name }}" + register: ec2_instances_async + async: 7200 + poll: 0 + + - name: Wait for instance destruction to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ ec2_instances_async.results }}" + loop_control: + index_var: index + label: "{{ platforms[index].name }}" + register: ec2_instances + until: ec2_instances is finished + retries: 300 + + - name: Destroy ephemeral security groups (if needed) + amazon.aws.ec2_security_group: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}" + name: "{{ item.security_group_name }}" + state: absent + vars: + vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}" + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.security_groups | length == 0 + + - name: Destroy ephemeral keys (if needed) + amazon.aws.ec2_key: + profile: "{{ item.aws_profile | default(omit) }}" + region: "{{ item.region | default(omit) }}" + name: "{{ item.key_name }}" + state: absent + loop: "{{ platforms }}" + loop_control: + index_var: index + label: "{{ item.name }}" + when: item.key_inject_method == "ec2" + + - name: Write Molecule instance configs + ansible.builtin.copy: + dest: "{{ molecule_instance_config }}" + content: "{{ {} | to_yaml }}" diff --git a/roles/prereq_zookeeper/molecule/default/molecule.yml b/roles/prereq_zookeeper/molecule/default/molecule.yml new file mode 100644 index 00000000..8d3ac246 --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/molecule.yml @@ -0,0 +1,49 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Requires a preexisting AWS VPC and subnet, the latter referenced by the +# environment variable, ROLE_SUBNET_ID, typically set in the Molecule ENV file +# which is /.env.yml + +# Requires AWS credentials, including region + +# Requires the following Python libraries on localhost +# - netaddr +# - boto3 + +driver: + name: ec2 +platforms: + - name: rhel9-4.molecule.internal + image_owner: "309956199498" + image_name: RHEL-9.4.0_HVM-* + instance_type: t3.medium + boot_wait_seconds: 15 + vpc_subnet_id: ${TEST_VPC_SUBNET_ID} + tags: + Name: molecule-prereq_zookeeper-rhel9-4 + Project: Molecule testing for prereq_zookeeper +provisioner: + name: ansible + inventory: + group_vars: + all: + tls_keystore_path: /tmp/tls_keystore_path + tls_keystore_path_generic: /tmp/tls_keystore_path_generic + tls_key_path: /tmp/tls_key_path + tls_key_path_generic: /tmp/tls_key_path_generic + tls_key_password_file: /tmp/tls_key_password_file + tls_key_path_plaintext: /tmp/tls_key_path_plaintext + tls_key_path_plaintext_generic: /tmp/tls_key_path_plaintext_generic diff --git a/roles/prereq_zookeeper/molecule/default/prepare.yml b/roles/prereq_zookeeper/molecule/default/prepare.yml new file mode 100644 index 00000000..54ed0303 --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/prepare.yml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Prepare + hosts: all + gather_facts: false + become: true + tasks: + - name: Create TLS paths + when: lookup('vars', tls_file, default=False) + ansible.builtin.file: + path: "{{ lookup('vars', tls_file) }}" + state: touch + owner: root + group: root + mode: "0640" + loop: + - tls_keystore_path + - tls_keystore_path_generic + - tls_key_path + - tls_key_path_generic + - tls_key_password_file + - tls_key_path_plaintext + - tls_key_path_plaintext_generic + loop_control: + loop_var: tls_file diff --git a/roles/prereq_zookeeper/molecule/default/requirements.yml b/roles/prereq_zookeeper/molecule/default/requirements.yml new file mode 100644 index 00000000..0d4d942e --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/requirements.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +collections: + - community.crypto + - amazon.aws + - ansible.posix + - ansible.utils + - community.general diff --git a/roles/prereq_zookeeper/molecule/default/verify.yml b/roles/prereq_zookeeper/molecule/default/verify.yml new file mode 100644 index 00000000..52fdfcb6 --- /dev/null +++ b/roles/prereq_zookeeper/molecule/default/verify.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify + hosts: all + gather_facts: false + vars_files: "../../vars/main.yml" + tasks: + - name: Check for zookeeper users + ansible.builtin.command: "grep {{ account.user }} /etc/passwd" + register: result + failed_when: result.rc != 0 + changed_when: false + loop: "{{ zookeeper_local_accounts }}" + loop_control: + loop_var: account + label: "{{ account.user }}" + + - name: Check TLS ACLs + ansible.builtin.include_role: + name: cloudera.exe.prereq_tls_acls + tasks_from: validate.yml + vars: + acl_user_accounts: "{{ zookeeper_local_accounts }}" diff --git a/roles/prereq_zookeeper/tasks/main.yml b/roles/prereq_zookeeper/tasks/main.yml new file mode 100644 index 00000000..55b6c27f --- /dev/null +++ b/roles/prereq_zookeeper/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create local user accounts + ansible.builtin.import_role: + name: cloudera.exe.prereq_local_account + vars: + local_accounts: "{{ zookeeper_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - home + - uid + - shell + - comment + - extra_groups + +- name: Set local user ACLs on TLS entities + ansible.builtin.import_role: + name: cloudera.exe.prereq_tls_acls + vars: + acl_user_accounts: "{{ zookeeper_local_accounts | community.general.keep_keys(target=target_keys) | list }}" + target_keys: + - user + - key_acl + - key_password_acl + - keystore_acl + - unencrypted_key_acl diff --git a/roles/prereq_zookeeper/vars/main.yml b/roles/prereq_zookeeper/vars/main.yml new file mode 100644 index 00000000..53a709d0 --- /dev/null +++ b/roles/prereq_zookeeper/vars/main.yml @@ -0,0 +1,20 @@ +--- +# Copyright 2024 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +zookeeper_local_accounts: + - user: zookeeper + home: /var/lib/zookeeper + comment: Zookeeper + keystore_acl: true From aebb99f3c99d5a14c55174a03d3c126469b3ba13 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:30 -0400 Subject: [PATCH 70/72] Update argument_specs to remove 'no_log' from documentation Signed-off-by: Webster Mudge --- roles/prereq_activitymonitor/meta/argument_specs.yml | 1 - roles/prereq_cm_database/meta/argument_specs.yml | 1 - roles/prereq_database/meta/argument_specs.yml | 4 +--- roles/prereq_dataviz_database/meta/argument_specs.yml | 1 - roles/prereq_hive_database/meta/argument_specs.yml | 1 - roles/prereq_hue_database/meta/argument_specs.yml | 1 - roles/prereq_knox_database/meta/argument_specs.yml | 1 - roles/prereq_oozie_database/meta/argument_specs.yml | 1 - roles/prereq_query_processor_database/meta/argument_specs.yml | 1 - roles/prereq_ranger_database/meta/argument_specs.yml | 1 - roles/prereq_reportsmanager/meta/argument_specs.yml | 1 - roles/prereq_schemaregistry_database/meta/argument_specs.yml | 1 - roles/prereq_smm_database/meta/argument_specs.yml | 1 - roles/prereq_ssb_database/meta/argument_specs.yml | 1 - 14 files changed, 1 insertion(+), 16 deletions(-) diff --git a/roles/prereq_activitymonitor/meta/argument_specs.yml b/roles/prereq_activitymonitor/meta/argument_specs.yml index 5a6d4556..e4996f70 100644 --- a/roles/prereq_activitymonitor/meta/argument_specs.yml +++ b/roles/prereq_activitymonitor/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true activity_monitor_username: description: The username for the Activity Monitor database user and owner of the database. type: str diff --git a/roles/prereq_cm_database/meta/argument_specs.yml b/roles/prereq_cm_database/meta/argument_specs.yml index 147f7fce..f2a6cdee 100644 --- a/roles/prereq_cm_database/meta/argument_specs.yml +++ b/roles/prereq_cm_database/meta/argument_specs.yml @@ -50,7 +50,6 @@ argument_specs: description: The password for the database admin login. type: str required: true - no_log: true database_host: description: The hostname or IP address of the database server. type: str diff --git a/roles/prereq_database/meta/argument_specs.yml b/roles/prereq_database/meta/argument_specs.yml index ca8c4bd9..ba635858 100644 --- a/roles/prereq_database/meta/argument_specs.yml +++ b/roles/prereq_database/meta/argument_specs.yml @@ -36,7 +36,6 @@ argument_specs: description: The password for the database admin login. type: str required: true - no_log: true database_host: description: The hostname or IP address of the database server. type: str @@ -63,8 +62,7 @@ argument_specs: description: The password for the database user. type: str required: true - no_log: true owner: - description: The name of the database user owning the database. Defaults to O(user). + description: The name of the database user owning the database. Defaults to O(database_accounts.user). type: str required: false diff --git a/roles/prereq_dataviz_database/meta/argument_specs.yml b/roles/prereq_dataviz_database/meta/argument_specs.yml index 3d0a610f..b7470cf8 100644 --- a/roles/prereq_dataviz_database/meta/argument_specs.yml +++ b/roles/prereq_dataviz_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true dataviz_username: description: The username for the Dataviz database user and owner of the database. type: str diff --git a/roles/prereq_hive_database/meta/argument_specs.yml b/roles/prereq_hive_database/meta/argument_specs.yml index 1fdb00f3..8858508c 100644 --- a/roles/prereq_hive_database/meta/argument_specs.yml +++ b/roles/prereq_hive_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true hive_username: description: The username for the Hive database user and owner of the database. type: str diff --git a/roles/prereq_hue_database/meta/argument_specs.yml b/roles/prereq_hue_database/meta/argument_specs.yml index a90a8304..d9386db8 100644 --- a/roles/prereq_hue_database/meta/argument_specs.yml +++ b/roles/prereq_hue_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true hue_username: description: The username for the Hue database user and owner of the database. type: str diff --git a/roles/prereq_knox_database/meta/argument_specs.yml b/roles/prereq_knox_database/meta/argument_specs.yml index 84fd9b2a..802ffb63 100644 --- a/roles/prereq_knox_database/meta/argument_specs.yml +++ b/roles/prereq_knox_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true knox_username: description: The username for the Knox database user and owner of the database. type: str diff --git a/roles/prereq_oozie_database/meta/argument_specs.yml b/roles/prereq_oozie_database/meta/argument_specs.yml index 1b4b58a3..24360918 100644 --- a/roles/prereq_oozie_database/meta/argument_specs.yml +++ b/roles/prereq_oozie_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true oozie_username: description: The username for the Oozie database user and owner of the database. type: str diff --git a/roles/prereq_query_processor_database/meta/argument_specs.yml b/roles/prereq_query_processor_database/meta/argument_specs.yml index 6dc7d1e0..9307b78c 100644 --- a/roles/prereq_query_processor_database/meta/argument_specs.yml +++ b/roles/prereq_query_processor_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true query_processor_username: description: The username for the Query Processor database user and owner of the database. type: str diff --git a/roles/prereq_ranger_database/meta/argument_specs.yml b/roles/prereq_ranger_database/meta/argument_specs.yml index f079e1b1..4bec0156 100644 --- a/roles/prereq_ranger_database/meta/argument_specs.yml +++ b/roles/prereq_ranger_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true ranger_username: description: The username for the Ranger database user and owner of the database. type: str diff --git a/roles/prereq_reportsmanager/meta/argument_specs.yml b/roles/prereq_reportsmanager/meta/argument_specs.yml index 9af1b5d9..f17a101c 100644 --- a/roles/prereq_reportsmanager/meta/argument_specs.yml +++ b/roles/prereq_reportsmanager/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true reports_manager_username: description: The username for the Reports Manager database user and owner of the database. type: str diff --git a/roles/prereq_schemaregistry_database/meta/argument_specs.yml b/roles/prereq_schemaregistry_database/meta/argument_specs.yml index 7236f747..8e670732 100644 --- a/roles/prereq_schemaregistry_database/meta/argument_specs.yml +++ b/roles/prereq_schemaregistry_database/meta/argument_specs.yml @@ -36,7 +36,6 @@ argument_specs: description: The password for the database admin login. type: str required: true - no_log: true database_host: description: The hostname or IP address of the database server. type: str diff --git a/roles/prereq_smm_database/meta/argument_specs.yml b/roles/prereq_smm_database/meta/argument_specs.yml index 6feb5abd..274e8626 100644 --- a/roles/prereq_smm_database/meta/argument_specs.yml +++ b/roles/prereq_smm_database/meta/argument_specs.yml @@ -40,7 +40,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true smm_username: description: The username for the Streams Messaging Manager database user and owner of the database. type: str diff --git a/roles/prereq_ssb_database/meta/argument_specs.yml b/roles/prereq_ssb_database/meta/argument_specs.yml index 874c7806..ab06459f 100644 --- a/roles/prereq_ssb_database/meta/argument_specs.yml +++ b/roles/prereq_ssb_database/meta/argument_specs.yml @@ -42,7 +42,6 @@ argument_specs: description: The password for the database administrative user. type: str required: true - no_log: true ssb_admin_username: description: The username for the SQL Stream Builder database user and owner of the database. type: str From e30bce85586f3be9f012ac1f3f0fea978500f34e Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:31 -0400 Subject: [PATCH 71/72] Update short descriptions for clarity in indices Signed-off-by: Webster Mudge --- roles/prereq_accumulo/meta/argument_specs.yml | 4 ++-- roles/prereq_activitymonitor/meta/argument_specs.yml | 4 ++-- roles/prereq_activitymonitor/tasks/main.yml | 1 + roles/prereq_atlas/README.md | 2 +- roles/prereq_atlas/meta/argument_specs.yml | 4 ++-- roles/prereq_cloudera_manager/meta/argument_specs.yml | 2 +- roles/prereq_cm_database/meta/argument_specs.yml | 2 +- roles/prereq_dataviz/meta/argument_specs.yml | 2 +- roles/prereq_druid/README.md | 2 +- roles/prereq_druid/meta/argument_specs.yml | 4 ++-- roles/prereq_ecs/README.md | 2 +- roles/prereq_ecs/meta/argument_specs.yml | 7 ++++--- roles/prereq_firewall/meta/argument_specs.yml | 2 +- roles/prereq_flink/meta/argument_specs.yml | 4 ++-- roles/prereq_flume/meta/argument_specs.yml | 4 ++-- roles/prereq_hadoop/meta/argument_specs.yml | 4 ++-- roles/prereq_hbase/README.md | 2 +- roles/prereq_hbase/meta/argument_specs.yml | 4 ++-- roles/prereq_hive/meta/argument_specs.yml | 4 ++-- roles/prereq_hive_database/README.md | 2 +- roles/prereq_hive_database/meta/argument_specs.yml | 2 +- roles/prereq_httpfs/meta/argument_specs.yml | 2 +- roles/prereq_hue/meta/argument_specs.yml | 2 +- roles/prereq_impala/meta/argument_specs.yml | 2 +- roles/prereq_jdk/meta/argument_specs.yml | 2 +- roles/prereq_kafka/meta/argument_specs.yml | 4 ++-- roles/prereq_kerberos/meta/argument_specs.yml | 2 +- roles/prereq_kernel/meta/argument_specs.yml | 2 +- roles/prereq_keytrustee/meta/argument_specs.yml | 2 +- roles/prereq_kms/meta/argument_specs.yml | 2 +- roles/prereq_knox/meta/argument_specs.yml | 2 +- roles/prereq_kudu/meta/argument_specs.yml | 2 +- roles/prereq_livy/meta/argument_specs.yml | 2 +- roles/prereq_mapreduce/meta/argument_specs.yml | 2 +- roles/prereq_nifi/meta/argument_specs.yml | 4 ++-- roles/prereq_nifiregistry/meta/argument_specs.yml | 2 +- roles/prereq_ntp/meta/argument_specs.yml | 2 +- roles/prereq_oozie/meta/argument_specs.yml | 2 +- roles/prereq_os/meta/argument_specs.yml | 2 +- roles/prereq_phoenix/meta/argument_specs.yml | 2 +- roles/prereq_psycopg2/meta/argument_specs.yml | 2 +- roles/prereq_python/meta/argument_specs.yml | 2 +- roles/prereq_ranger/meta/argument_specs.yml | 2 +- roles/prereq_rngd/meta/argument_specs.yml | 2 +- roles/prereq_schemaregistry/meta/argument_specs.yml | 2 +- roles/prereq_selinux/meta/argument_specs.yml | 2 +- roles/prereq_sentry/meta/argument_specs.yml | 2 +- roles/prereq_services/meta/argument_specs.yml | 2 +- roles/prereq_smm/meta/argument_specs.yml | 2 +- roles/prereq_solr/meta/argument_specs.yml | 2 +- roles/prereq_spark/meta/argument_specs.yml | 2 +- roles/prereq_spark2/meta/argument_specs.yml | 2 +- roles/prereq_sqoop/meta/argument_specs.yml | 2 +- roles/prereq_ssb/meta/argument_specs.yml | 2 +- roles/prereq_superset/meta/argument_specs.yml | 2 +- roles/prereq_supported/meta/argument_specs.yml | 2 +- roles/prereq_thp/meta/argument_specs.yml | 2 +- roles/prereq_yarn/meta/argument_specs.yml | 2 +- roles/prereq_zeppelin/meta/argument_specs.yml | 2 +- 59 files changed, 73 insertions(+), 71 deletions(-) diff --git a/roles/prereq_accumulo/meta/argument_specs.yml b/roles/prereq_accumulo/meta/argument_specs.yml index b6f5e983..8d8a5e1c 100644 --- a/roles/prereq_accumulo/meta/argument_specs.yml +++ b/roles/prereq_accumulo/meta/argument_specs.yml @@ -15,8 +15,8 @@ argument_specs: main: - short_description: Set up for Accumulo + short_description: Set up user accounts for Accumulo description: | - Set up for Accumulo usage, notably, create the local C(accumulo) user. + Set up for Apache Accumulo usage, notably, create the local C(accumulo) user. author: Cloudera Labs options: {} diff --git a/roles/prereq_activitymonitor/meta/argument_specs.yml b/roles/prereq_activitymonitor/meta/argument_specs.yml index e4996f70..9de4a75d 100644 --- a/roles/prereq_activitymonitor/meta/argument_specs.yml +++ b/roles/prereq_activitymonitor/meta/argument_specs.yml @@ -18,8 +18,8 @@ argument_specs: short_description: Set up database and user accounts for Activity Monitor description: - Set up the Activity Monitor database and its associated user accounts, ensuring proper configuration for Activity Monitor operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the `activity_monitor_username`, - `activity_monitor_password`, and `activity_monitor_database` variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(activity_monitor_username), + O(activity_monitor_password), and O(activity_monitor_database) variables. author: Cloudera Labs options: database_type: diff --git a/roles/prereq_activitymonitor/tasks/main.yml b/roles/prereq_activitymonitor/tasks/main.yml index 64713967..d183d422 100644 --- a/roles/prereq_activitymonitor/tasks/main.yml +++ b/roles/prereq_activitymonitor/tasks/main.yml @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + - name: Provision databases and user accounts ansible.builtin.import_role: name: cloudera.exe.prereq_database diff --git a/roles/prereq_atlas/README.md b/roles/prereq_atlas/README.md index fdbdc8d0..02125cf2 100644 --- a/roles/prereq_atlas/README.md +++ b/roles/prereq_atlas/README.md @@ -2,7 +2,7 @@ Set up for Atlas -This role prepares a host for Atlas usage by creating a dedicated system user and group named `atlas`. This user is essential for running Atlas processes with appropriate permissions and isolation within a Hadoop environment. +This role prepares a host for Apache Atlas usage by creating a dedicated system user and group named `atlas`. This user is essential for running Atlas processes with appropriate permissions and isolation within a Hadoop environment. The role will: - Create the `atlas` system user and group. diff --git a/roles/prereq_atlas/meta/argument_specs.yml b/roles/prereq_atlas/meta/argument_specs.yml index ad33beaf..8699a424 100644 --- a/roles/prereq_atlas/meta/argument_specs.yml +++ b/roles/prereq_atlas/meta/argument_specs.yml @@ -15,8 +15,8 @@ argument_specs: main: - short_description: Set up for Atlas + short_description: Set up user accounts for Atlas description: | - Set up for Hadoop usage, notably, create the local C(atlas) user. + Set up for Apache Atlas usage, notably, create the local C(atlas) user. author: Cloudera Labs options: {} diff --git a/roles/prereq_cloudera_manager/meta/argument_specs.yml b/roles/prereq_cloudera_manager/meta/argument_specs.yml index e3426b0b..61141b0f 100644 --- a/roles/prereq_cloudera_manager/meta/argument_specs.yml +++ b/roles/prereq_cloudera_manager/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Cloudera Manager + short_description: Set up user accounts and LDAP for Kerberos for Cloudera Manager description: | Set up for Cloudera Manager usage, notably, create the local C(cloudera-scm) user, including C(HOME) directory permissions. Set up TLS ACLs, if needed. diff --git a/roles/prereq_cm_database/meta/argument_specs.yml b/roles/prereq_cm_database/meta/argument_specs.yml index f2a6cdee..9a5fd62c 100644 --- a/roles/prereq_cm_database/meta/argument_specs.yml +++ b/roles/prereq_cm_database/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Database and user for Cloudera Manager" + short_description: Set up database and user accounts for Cloudera Manager description: - Creates the database and user for Cloudera Manager on PostgreSQL author: diff --git a/roles/prereq_dataviz/meta/argument_specs.yml b/roles/prereq_dataviz/meta/argument_specs.yml index 8f4d21ef..b69dab7c 100644 --- a/roles/prereq_dataviz/meta/argument_specs.yml +++ b/roles/prereq_dataviz/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Dataviz + short_description: Set up user accounts for Dataviz description: | Set up for Dataviz usage, notably, create the local C(dataviz) user. author: Cloudera Labs diff --git a/roles/prereq_druid/README.md b/roles/prereq_druid/README.md index 3d9b4fd3..f5ec9109 100644 --- a/roles/prereq_druid/README.md +++ b/roles/prereq_druid/README.md @@ -1,6 +1,6 @@ # prereq_druid -Set up for Apache Druid +Set up for Druid This role prepares a host for Apache Druid usage by creating a dedicated system user and group named `druid`. This user is essential for running Apache Druid processes with appropriate permissions and isolation. diff --git a/roles/prereq_druid/meta/argument_specs.yml b/roles/prereq_druid/meta/argument_specs.yml index d1b3105a..8fd7fe4d 100644 --- a/roles/prereq_druid/meta/argument_specs.yml +++ b/roles/prereq_druid/meta/argument_specs.yml @@ -15,8 +15,8 @@ argument_specs: main: - short_description: Set up for Druid + short_description: Set up user accounts for Druid description: | - Set up for Druid usage, notably, create the local C(druid) user. + Set up for Apache Druid usage, notably, create the local C(druid) user. author: Cloudera Labs options: {} diff --git a/roles/prereq_ecs/README.md b/roles/prereq_ecs/README.md index 492424b4..8434d192 100644 --- a/roles/prereq_ecs/README.md +++ b/roles/prereq_ecs/README.md @@ -2,7 +2,7 @@ Set up for ECS -This role prepares a host for Cloudera's ECS (Embedded Container Service) usage by creating the required local users and configuring the firewall and network settings. It ensures that the host's environment is properly configured to support ECS components and operations, including user permissions and network security rules. +This role prepares a host for Cloudera Embedded Container Service (ECS) usage by creating the required local users and configuring the firewall and network settings. It ensures that the host's environment is properly configured to support ECS components and operations, including user permissions and network security rules. The role will: - Create the necessary system users and groups for ECS, based on a list provided by the `prereq_cloudera_manager` role. diff --git a/roles/prereq_ecs/meta/argument_specs.yml b/roles/prereq_ecs/meta/argument_specs.yml index 654d04ed..2321af5f 100644 --- a/roles/prereq_ecs/meta/argument_specs.yml +++ b/roles/prereq_ecs/meta/argument_specs.yml @@ -15,9 +15,10 @@ argument_specs: main: - short_description: Set up for ECS + short_description: Set up user accounts, firewall, and networking for ECS description: - - Set up for ECS usage including creation of required local users and configuration of firewall and networking configuration. - - The list of local users are taken from the M(cloudera.exe.prereq_cloudera_manager) role. + - Set up for Cloudera Embedded Container Service (ECS) usage including creation of required local users and configuration + of firewall and networking configuration. + - The list of local users are taken from the P(cloudera.exe.prereq_cloudera_manager#role) role. author: Cloudera Labs options: {} diff --git a/roles/prereq_firewall/meta/argument_specs.yml b/roles/prereq_firewall/meta/argument_specs.yml index 4a04df09..75346987 100644 --- a/roles/prereq_firewall/meta/argument_specs.yml +++ b/roles/prereq_firewall/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Disable firewalls + short_description: Disable firewalls for a deployment description: - Disable firewalls, if installed, and optionally back up iptable rules. - Backup files include all iptable tables for both IPv4 and IPv6. diff --git a/roles/prereq_flink/meta/argument_specs.yml b/roles/prereq_flink/meta/argument_specs.yml index c1426409..010ca01e 100644 --- a/roles/prereq_flink/meta/argument_specs.yml +++ b/roles/prereq_flink/meta/argument_specs.yml @@ -15,9 +15,9 @@ argument_specs: main: - short_description: Set up for Flink + short_description: Set up user accounts for Flink description: | - Set up for Flink usage, notably, create the local C(flink) user. + Set up for Apache Flink usage, notably, create the local C(flink) user. Optionally, set up ACLs on TLS entities. author: Cloudera Labs options: {} diff --git a/roles/prereq_flume/meta/argument_specs.yml b/roles/prereq_flume/meta/argument_specs.yml index f8bfe135..b4ef32f1 100644 --- a/roles/prereq_flume/meta/argument_specs.yml +++ b/roles/prereq_flume/meta/argument_specs.yml @@ -15,9 +15,9 @@ argument_specs: main: - short_description: Set up for Flume + short_description: Set up user accounts for Flume description: | - Set up for Flume usage, notably, create the local C(flume) user. + Set up for Apache Flume usage, notably, create the local C(flume) user. Optionally, set up ACLs on TLS entities. author: Cloudera Labs options: {} diff --git a/roles/prereq_hadoop/meta/argument_specs.yml b/roles/prereq_hadoop/meta/argument_specs.yml index ced1a8a7..9c9a6f1d 100644 --- a/roles/prereq_hadoop/meta/argument_specs.yml +++ b/roles/prereq_hadoop/meta/argument_specs.yml @@ -15,8 +15,8 @@ argument_specs: main: - short_description: Set up for Hadoop + short_description: Set up user accounts for Hadoop description: | - Set up for Hadoop usage, notably, create the local C(hadoop) user group. + Set up for Apache Hadoop usage, notably, create the local C(hadoop) user group. author: Cloudera Labs options: {} diff --git a/roles/prereq_hbase/README.md b/roles/prereq_hbase/README.md index e362e71e..4e2732f0 100644 --- a/roles/prereq_hbase/README.md +++ b/roles/prereq_hbase/README.md @@ -1,6 +1,6 @@ # prereq_hbase -Set up for Hbase +Set up for HBase This role prepares a host for Apache HBase usage by creating a dedicated system user and group named `hbase`. This user is essential for running HBase processes with appropriate permissions and isolation. The role can also optionally set up Access Control Lists (ACLs) on TLS entities if required for secure HBase communication. diff --git a/roles/prereq_hbase/meta/argument_specs.yml b/roles/prereq_hbase/meta/argument_specs.yml index ddd9bd0b..730636a8 100644 --- a/roles/prereq_hbase/meta/argument_specs.yml +++ b/roles/prereq_hbase/meta/argument_specs.yml @@ -15,9 +15,9 @@ argument_specs: main: - short_description: Set up for Hbase + short_description: Set up user accounts for HBase description: | - Set up for Hbase usage, notably, create the local C(hbase) user. + Set up for Apache HBase usage, notably, create the local C(hbase) user. Optionally, set up ACLs on TLS entities. author: Cloudera Labs options: {} diff --git a/roles/prereq_hive/meta/argument_specs.yml b/roles/prereq_hive/meta/argument_specs.yml index 29adc96b..2250557d 100644 --- a/roles/prereq_hive/meta/argument_specs.yml +++ b/roles/prereq_hive/meta/argument_specs.yml @@ -15,9 +15,9 @@ argument_specs: main: - short_description: Set up for Hive + short_description: Set up user accounts for Hive description: | - Set up for Hive usage, notably, create the local C(hive) user. + Set up for Apache Hive usage, notably, create the local C(hive) user. Optionally, set up ACLs on TLS entities. author: Cloudera Labs options: {} diff --git a/roles/prereq_hive_database/README.md b/roles/prereq_hive_database/README.md index 0ea99fe8..b70d7c84 100644 --- a/roles/prereq_hive_database/README.md +++ b/roles/prereq_hive_database/README.md @@ -2,7 +2,7 @@ Set up database and user accounts for Hive -This role automates the setup of a database and its associated user accounts specifically for Hive services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. +This role automates the setup of a database and its associated user accounts specifically for Apache Hive services. It supports various database types, including PostgreSQL, MySQL, and Oracle. The role creates the database and a dedicated user with ownership privileges, using sensible defaults that can be easily overridden. The role will: - Connect to the specified database server using administrative credentials. diff --git a/roles/prereq_hive_database/meta/argument_specs.yml b/roles/prereq_hive_database/meta/argument_specs.yml index 8858508c..3986cae1 100644 --- a/roles/prereq_hive_database/meta/argument_specs.yml +++ b/roles/prereq_hive_database/meta/argument_specs.yml @@ -17,7 +17,7 @@ argument_specs: main: short_description: Set up database and user accounts for Hive description: - - Set up the Hive database and its associated user accounts, ensuring proper configuration for Hive operations. + - Set up the Apache Hive database and its associated user accounts, ensuring proper configuration for Hive operations. - Default values are used for the username, password, and database name, but these can be overwritten by setting the `hive_username`, `hive_password`, and `hive_database` variables. author: Cloudera Labs diff --git a/roles/prereq_httpfs/meta/argument_specs.yml b/roles/prereq_httpfs/meta/argument_specs.yml index d8bafcb1..313dad0e 100644 --- a/roles/prereq_httpfs/meta/argument_specs.yml +++ b/roles/prereq_httpfs/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for HttpFS + short_description: Set up user accounts for HttpFS description: | Set up for HttpFS usage, notably, create the local C(httpfs) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_hue/meta/argument_specs.yml b/roles/prereq_hue/meta/argument_specs.yml index 7fb4cdde..b684ed4d 100644 --- a/roles/prereq_hue/meta/argument_specs.yml +++ b/roles/prereq_hue/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Hue + short_description: Set up user accounts and Kerberos for Hue description: | Set up for Hue usage, notably, create the local C(hue) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_impala/meta/argument_specs.yml b/roles/prereq_impala/meta/argument_specs.yml index 3eba037d..c1c549cb 100644 --- a/roles/prereq_impala/meta/argument_specs.yml +++ b/roles/prereq_impala/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Impala + short_description: Set up user accounts for Impala description: | Set up for Impala usage, notably, create the local C(impala) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_jdk/meta/argument_specs.yml b/roles/prereq_jdk/meta/argument_specs.yml index 1d17fcee..feb96895 100644 --- a/roles/prereq_jdk/meta/argument_specs.yml +++ b/roles/prereq_jdk/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up JDK + short_description: Set up the JDK description: - Set up the Java Development Kit (JDK), optionally installing the JDK itself. - For JDK 9 and below, optionally enable the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy. diff --git a/roles/prereq_kafka/meta/argument_specs.yml b/roles/prereq_kafka/meta/argument_specs.yml index 8bf6c914..e36e9826 100644 --- a/roles/prereq_kafka/meta/argument_specs.yml +++ b/roles/prereq_kafka/meta/argument_specs.yml @@ -15,8 +15,8 @@ argument_specs: main: - short_description: Set up for Kafka + short_description: Set up user accounts for Kafka description: | - Set up for Kafka usage, notably, create the local C(kafka) and C(cruisecontrol) users. + Set up for Apache Kafka usage, notably, create the local C(kafka) and C(cruisecontrol) users. author: Cloudera Labs options: {} diff --git a/roles/prereq_kerberos/meta/argument_specs.yml b/roles/prereq_kerberos/meta/argument_specs.yml index 7c87e94e..956a5d44 100644 --- a/roles/prereq_kerberos/meta/argument_specs.yml +++ b/roles/prereq_kerberos/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Kerberos + short_description: Set up Kerberos for deployments description: - Set up for Kerberos usage, including OS-specific Kerberos client libraries. - Configure Kerberos credential cache. diff --git a/roles/prereq_kernel/meta/argument_specs.yml b/roles/prereq_kernel/meta/argument_specs.yml index 88c99ed6..46f48243 100644 --- a/roles/prereq_kernel/meta/argument_specs.yml +++ b/roles/prereq_kernel/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Update kernel parameters + short_description: Update OS kernel parameters for deployments description: | Updates and applies kernel parameters using C(sysctl). This includes controlling memory management (swappiness and memory overcommit) diff --git a/roles/prereq_keytrustee/meta/argument_specs.yml b/roles/prereq_keytrustee/meta/argument_specs.yml index 8735250f..e3c695bb 100644 --- a/roles/prereq_keytrustee/meta/argument_specs.yml +++ b/roles/prereq_keytrustee/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Key Trustee + short_description: Set up user accounts for Key Trustee description: | Set up for Cloudera Navigator Key Trustee usage, notably, create the local C(keytrustee) users. author: Cloudera Labs diff --git a/roles/prereq_kms/meta/argument_specs.yml b/roles/prereq_kms/meta/argument_specs.yml index 6621294a..5d8471ca 100644 --- a/roles/prereq_kms/meta/argument_specs.yml +++ b/roles/prereq_kms/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for KMS + short_description: Set up user accounts for KMS description: | Set up for Cloudera Key Management System (KMS) usage, notably, create the local C(kms) users. author: Cloudera Labs diff --git a/roles/prereq_knox/meta/argument_specs.yml b/roles/prereq_knox/meta/argument_specs.yml index f816cda1..22d30701 100644 --- a/roles/prereq_knox/meta/argument_specs.yml +++ b/roles/prereq_knox/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Knox + short_description: Set up user accounts for Knox description: | Set up for Apache Knox usage, notably, create the local C(knox) user. author: Cloudera Labs diff --git a/roles/prereq_kudu/meta/argument_specs.yml b/roles/prereq_kudu/meta/argument_specs.yml index 543cc78f..35bfd5eb 100644 --- a/roles/prereq_kudu/meta/argument_specs.yml +++ b/roles/prereq_kudu/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Kudu + short_description: Set up user accounts for Kudu description: | Set up for Apache Kudu usage, notably, create the local C(kudu) user. author: Cloudera Labs diff --git a/roles/prereq_livy/meta/argument_specs.yml b/roles/prereq_livy/meta/argument_specs.yml index 5fd8be89..a1ef4283 100644 --- a/roles/prereq_livy/meta/argument_specs.yml +++ b/roles/prereq_livy/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Livy + short_description: Set up user accounts for Livy description: | Set up for Apache Livy usage, notably, create the local C(livy) user. author: Cloudera Labs diff --git a/roles/prereq_mapreduce/meta/argument_specs.yml b/roles/prereq_mapreduce/meta/argument_specs.yml index d52f96a8..d86248e0 100644 --- a/roles/prereq_mapreduce/meta/argument_specs.yml +++ b/roles/prereq_mapreduce/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for MapReduce + short_description: Set up user accounts for MapReduce description: | Set up for MapReduce usage, notably, create the local C(mapred) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_nifi/meta/argument_specs.yml b/roles/prereq_nifi/meta/argument_specs.yml index 25147249..17a847f1 100644 --- a/roles/prereq_nifi/meta/argument_specs.yml +++ b/roles/prereq_nifi/meta/argument_specs.yml @@ -15,9 +15,9 @@ argument_specs: main: - short_description: Set up for Nifi + short_description: Set up user accounts for NiFi description: | - Set up for Apache Nifi usage, notably, create the local C(nifi) user. + Set up for Apache NiFi usage, notably, create the local C(nifi) user. Optionally, set up ACLs on TLS entities. author: Cloudera Labs options: {} diff --git a/roles/prereq_nifiregistry/meta/argument_specs.yml b/roles/prereq_nifiregistry/meta/argument_specs.yml index 83350834..e19f3182 100644 --- a/roles/prereq_nifiregistry/meta/argument_specs.yml +++ b/roles/prereq_nifiregistry/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for NiFi Registry + short_description: Set up user accounts for NiFi Registry description: | Set up for Apache NiFi Registry usage, notably, create the local C(nifiregistry) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_ntp/meta/argument_specs.yml b/roles/prereq_ntp/meta/argument_specs.yml index 37711504..672e331d 100644 --- a/roles/prereq_ntp/meta/argument_specs.yml +++ b/roles/prereq_ntp/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Manage NTP Services + short_description: Set up NTP services for deployments description: | This module installs the Chrony NTP package if neither Chrony nor NTP is currently installed or running on the system. If both services are present and running, it will stop the NTP service to prioritize Chrony. diff --git a/roles/prereq_oozie/meta/argument_specs.yml b/roles/prereq_oozie/meta/argument_specs.yml index 35f5cdb8..fbef2e24 100644 --- a/roles/prereq_oozie/meta/argument_specs.yml +++ b/roles/prereq_oozie/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Oozie + short_description: Set up user accounts for Oozie description: | Set up for Apache Oozie usage, notably, create the local C(oozie) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_os/meta/argument_specs.yml b/roles/prereq_os/meta/argument_specs.yml index 3407e2e6..512c3c17 100644 --- a/roles/prereq_os/meta/argument_specs.yml +++ b/roles/prereq_os/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Update general OS requirements. + short_description: Update general OS requirements for deployments description: - Update timezone. - Update Cloudera Manager access to global C(/tmp) directory. diff --git a/roles/prereq_phoenix/meta/argument_specs.yml b/roles/prereq_phoenix/meta/argument_specs.yml index 881aaead..1e4119b9 100644 --- a/roles/prereq_phoenix/meta/argument_specs.yml +++ b/roles/prereq_phoenix/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Phoenix + short_description: Set up user accounts for Phoenix description: | Set up for Apache Phoenix usage, notably, create local C(phoenix) user. author: Cloudera Labs diff --git a/roles/prereq_psycopg2/meta/argument_specs.yml b/roles/prereq_psycopg2/meta/argument_specs.yml index dcdfb524..bc2d6889 100644 --- a/roles/prereq_psycopg2/meta/argument_specs.yml +++ b/roles/prereq_psycopg2/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Install psycopg2" + short_description: Install psycopg2 for PostgreSQL for deployments description: - Installs the psycopg2 Python package for PostgreSQL database. author: diff --git a/roles/prereq_python/meta/argument_specs.yml b/roles/prereq_python/meta/argument_specs.yml index ac638b51..b0c39f03 100644 --- a/roles/prereq_python/meta/argument_specs.yml +++ b/roles/prereq_python/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Install Python" + short_description: Install Python for deployments description: - Ensures that the supported versions of Python for specified versions of Cloudera Manager and Runtime. - Also ensures that Pip is installed and updated. diff --git a/roles/prereq_ranger/meta/argument_specs.yml b/roles/prereq_ranger/meta/argument_specs.yml index 3df01715..7e65ae8b 100644 --- a/roles/prereq_ranger/meta/argument_specs.yml +++ b/roles/prereq_ranger/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Ranger + short_description: Set up user accounts for Ranger description: | Set up for Apache Ranger usage, notably, create the local C(ranger,rangerraz,rangertagsync) users and local C(ranger) group. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_rngd/meta/argument_specs.yml b/roles/prereq_rngd/meta/argument_specs.yml index 5c59fd1f..93442c5b 100644 --- a/roles/prereq_rngd/meta/argument_specs.yml +++ b/roles/prereq_rngd/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Install the Random Number Generator package" + short_description: Install the Random Number Generator package for deployments description: - Installs and configures the Random Number Generator (rngd) package. author: diff --git a/roles/prereq_schemaregistry/meta/argument_specs.yml b/roles/prereq_schemaregistry/meta/argument_specs.yml index afaffb0c..0ff1a362 100644 --- a/roles/prereq_schemaregistry/meta/argument_specs.yml +++ b/roles/prereq_schemaregistry/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Schema Registry + short_description: Set up user accounts for Schema Registry description: | Set up for Schema Registry usage, notably, create the local C(schemaregistry) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_selinux/meta/argument_specs.yml b/roles/prereq_selinux/meta/argument_specs.yml index 98833ccd..650d227c 100644 --- a/roles/prereq_selinux/meta/argument_specs.yml +++ b/roles/prereq_selinux/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Manage SELinux policy enforcement. + short_description: Manage SELinux policy enforcement for deployments description: - Manage the enforcement of the SELinux. - Installs packages for SELinux management, if needed, but does not install SELinux. diff --git a/roles/prereq_sentry/meta/argument_specs.yml b/roles/prereq_sentry/meta/argument_specs.yml index 77e5f1ea..31b62688 100644 --- a/roles/prereq_sentry/meta/argument_specs.yml +++ b/roles/prereq_sentry/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Sentry + short_description: Set up user accounts for Sentry description: | Set up for Apache Sentry usage, notably, create the local C(sentry) user. author: Cloudera Labs diff --git a/roles/prereq_services/meta/argument_specs.yml b/roles/prereq_services/meta/argument_specs.yml index d5f3a1ec..a01d41a3 100644 --- a/roles/prereq_services/meta/argument_specs.yml +++ b/roles/prereq_services/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Manage operating system services" + short_description: Manage operating system services for deployments description: - Manage operating system services for Cloudera on premises deployment. - Includes installing and configuring required services as well as disabling unnecessary services. diff --git a/roles/prereq_smm/meta/argument_specs.yml b/roles/prereq_smm/meta/argument_specs.yml index fb8b555b..39138318 100644 --- a/roles/prereq_smm/meta/argument_specs.yml +++ b/roles/prereq_smm/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Streams Messaging Manager + short_description: Set up user accounts and directories for Streams Messaging Manager description: - Set up for Streams Messaging Manager usage, notably, create the local C(streamsmsgmgr) and C(streamsrepmgr) users. - Also creates a symbolic link of the Streams Messaging Manager user home directory required for the Custom Service Descriptor (CSD) installation. diff --git a/roles/prereq_solr/meta/argument_specs.yml b/roles/prereq_solr/meta/argument_specs.yml index 2248ba4d..ae8e1335 100644 --- a/roles/prereq_solr/meta/argument_specs.yml +++ b/roles/prereq_solr/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Solr + short_description: Set up user accounts for Solr description: | Set up for Apache Solr usage, notably, create the local C(solr) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_spark/meta/argument_specs.yml b/roles/prereq_spark/meta/argument_specs.yml index 390a6b25..cde65253 100644 --- a/roles/prereq_spark/meta/argument_specs.yml +++ b/roles/prereq_spark/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Spark + short_description: Set up user accounts for Spark description: | Set up for Apache Spark usage, notably, create the local C(spark) user and local C(spark) group. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_spark2/meta/argument_specs.yml b/roles/prereq_spark2/meta/argument_specs.yml index affa14d1..66aa981c 100644 --- a/roles/prereq_spark2/meta/argument_specs.yml +++ b/roles/prereq_spark2/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Spark2 + short_description: Set up user accounts for Spark2 description: | Set up for Apache Spark2 usage, notably, create the local C(spark2) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_sqoop/meta/argument_specs.yml b/roles/prereq_sqoop/meta/argument_specs.yml index bc700de7..2bde7e19 100644 --- a/roles/prereq_sqoop/meta/argument_specs.yml +++ b/roles/prereq_sqoop/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Sqoop + short_description: Set up user accounts for Sqoop description: | Set up for Apache Sqoop usage, notably, create the local C(sqoop,sqoop2) users and local C(sqoop) group. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_ssb/meta/argument_specs.yml b/roles/prereq_ssb/meta/argument_specs.yml index e6b54f95..5e17d926 100644 --- a/roles/prereq_ssb/meta/argument_specs.yml +++ b/roles/prereq_ssb/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for SSB + short_description: Set up user accounts for SSB description: | Set up for SQL Stream Builder (SSB) usage, notably, create the local C(ssb) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_superset/meta/argument_specs.yml b/roles/prereq_superset/meta/argument_specs.yml index cc2eca4f..36f2323a 100644 --- a/roles/prereq_superset/meta/argument_specs.yml +++ b/roles/prereq_superset/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Superset + short_description: Set up user accounts for Superset description: | Set up for Apache Superset usage, notably, create the local C(superset) user. author: Cloudera Labs diff --git a/roles/prereq_supported/meta/argument_specs.yml b/roles/prereq_supported/meta/argument_specs.yml index 76f3c791..8da5063a 100644 --- a/roles/prereq_supported/meta/argument_specs.yml +++ b/roles/prereq_supported/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: "Verify configuration against support matrix" + short_description: Verify configuration against support matrix description: - Verification of various system and configuration settings against the Cloudera on premise support matrix available at supportmatrix.cloudera.com. - Additionally, the I(support_matrix) variable defined in this role can be imported and used in other roles. diff --git a/roles/prereq_thp/meta/argument_specs.yml b/roles/prereq_thp/meta/argument_specs.yml index 15f0fa66..5b6ac89a 100644 --- a/roles/prereq_thp/meta/argument_specs.yml +++ b/roles/prereq_thp/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Disable Transparent Huge Pages + short_description: Disable Transparent Huge Pages for deployments description: | Disable Transparent Huge Pages (THP) and, optionally, rebuild the GRUB bootloader. author: Cloudera Labs diff --git a/roles/prereq_yarn/meta/argument_specs.yml b/roles/prereq_yarn/meta/argument_specs.yml index e6cef6c6..72f67fff 100644 --- a/roles/prereq_yarn/meta/argument_specs.yml +++ b/roles/prereq_yarn/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for YARN + short_description: Set up user accounts for YARN description: | Set up for Apache Hadoop YARN usage, notably, create the local C(yarn) user. Optionally, set up ACLs on TLS entities. diff --git a/roles/prereq_zeppelin/meta/argument_specs.yml b/roles/prereq_zeppelin/meta/argument_specs.yml index a6f44c34..97505359 100644 --- a/roles/prereq_zeppelin/meta/argument_specs.yml +++ b/roles/prereq_zeppelin/meta/argument_specs.yml @@ -15,7 +15,7 @@ argument_specs: main: - short_description: Set up for Zeppelin + short_description: Set up user accounts for Zeppelin description: | Set up for Apache Zeppelin usage, notably, create the local C(zeppelin) user. Optionally, set up ACLs on TLS entities. From 2852d916a4706616398d6d699e854f33adbd36a2 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Mon, 4 Aug 2025 14:43:31 -0400 Subject: [PATCH 72/72] Update documentation for release version and formatting Signed-off-by: Webster Mudge --- roles/prereq_accumulo/meta/argument_specs.yml | 5 +++-- roles/prereq_activitymonitor/meta/argument_specs.yml | 1 + roles/prereq_atlas/meta/argument_specs.yml | 5 +++-- roles/prereq_cloudera_manager/meta/argument_specs.yml | 9 +++++---- roles/prereq_cm_database/meta/argument_specs.yml | 3 ++- roles/prereq_database/meta/argument_specs.yml | 5 +++-- roles/prereq_dataviz/meta/argument_specs.yml | 5 +++-- roles/prereq_dataviz_database/meta/argument_specs.yml | 5 +++-- roles/prereq_druid/meta/argument_specs.yml | 5 +++-- roles/prereq_ecs/files/networkmanager.conf | 2 +- roles/prereq_ecs/meta/argument_specs.yml | 1 + roles/prereq_firewall/meta/argument_specs.yml | 1 + roles/prereq_flink/meta/argument_specs.yml | 7 ++++--- roles/prereq_flume/meta/argument_specs.yml | 7 ++++--- roles/prereq_hadoop/meta/argument_specs.yml | 5 +++-- roles/prereq_hbase/meta/argument_specs.yml | 7 ++++--- roles/prereq_hdfs/meta/argument_specs.yml | 7 ++++--- roles/prereq_hive/meta/argument_specs.yml | 7 ++++--- roles/prereq_hive_database/meta/argument_specs.yml | 1 + roles/prereq_httpfs/meta/argument_specs.yml | 7 ++++--- roles/prereq_hue/meta/argument_specs.yml | 9 +++++---- roles/prereq_hue_database/meta/argument_specs.yml | 1 + roles/prereq_impala/meta/argument_specs.yml | 7 ++++--- roles/prereq_jdk/library/jdk_facts.py | 2 +- roles/prereq_jdk/meta/argument_specs.yml | 1 + roles/prereq_kafka/meta/argument_specs.yml | 5 +++-- roles/prereq_kerberos/meta/argument_specs.yml | 1 + roles/prereq_kernel/meta/argument_specs.yml | 9 +++++---- roles/prereq_keytrustee/meta/argument_specs.yml | 5 +++-- roles/prereq_kms/meta/argument_specs.yml | 5 +++-- roles/prereq_knox/meta/argument_specs.yml | 5 +++-- roles/prereq_knox_database/meta/argument_specs.yml | 5 +++-- roles/prereq_kudu/meta/argument_specs.yml | 5 +++-- roles/prereq_livy/meta/argument_specs.yml | 5 +++-- roles/prereq_local_account/meta/argument_specs.yml | 5 +++-- roles/prereq_mapreduce/meta/argument_specs.yml | 7 ++++--- roles/prereq_network_dns/meta/argument_specs.yml | 7 ++++--- roles/prereq_nifi/meta/argument_specs.yml | 7 ++++--- roles/prereq_nifiregistry/meta/argument_specs.yml | 7 ++++--- roles/prereq_ntp/meta/argument_specs.yml | 7 ++++--- roles/prereq_oozie/meta/argument_specs.yml | 7 ++++--- roles/prereq_oozie_database/meta/argument_specs.yml | 1 + roles/prereq_os/meta/argument_specs.yml | 1 + roles/prereq_phoenix/meta/argument_specs.yml | 5 +++-- roles/prereq_psycopg2/meta/argument_specs.yml | 1 + roles/prereq_python/README.md | 2 +- roles/prereq_python/meta/argument_specs.yml | 1 + .../meta/argument_specs.yml | 5 +++-- roles/prereq_ranger/meta/argument_specs.yml | 7 ++++--- roles/prereq_ranger_database/meta/argument_specs.yml | 5 +++-- roles/prereq_reportsmanager/meta/argument_specs.yml | 5 +++-- roles/prereq_rngd/meta/argument_specs.yml | 1 + roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 | 2 +- roles/prereq_schemaregistry/meta/argument_specs.yml | 7 ++++--- .../meta/argument_specs.yml | 5 +++-- roles/prereq_selinux/meta/argument_specs.yml | 1 + roles/prereq_sentry/meta/argument_specs.yml | 5 +++-- roles/prereq_services/meta/argument_specs.yml | 1 + roles/prereq_smm/meta/argument_specs.yml | 1 + roles/prereq_smm_database/meta/argument_specs.yml | 8 +++++--- roles/prereq_smm_database/molecule/default/prepare.yml | 1 - .../molecule/default/requirements.yml | 1 - roles/prereq_smm_database/molecule/default/verify.yml | 1 - roles/prereq_smm_database/tasks/main.yml | 2 -- roles/prereq_solr/meta/argument_specs.yml | 7 ++++--- roles/prereq_spark/meta/argument_specs.yml | 7 ++++--- roles/prereq_spark2/meta/argument_specs.yml | 7 ++++--- roles/prereq_sqoop/meta/argument_specs.yml | 7 ++++--- roles/prereq_ssb/meta/argument_specs.yml | 7 ++++--- roles/prereq_ssb_database/meta/argument_specs.yml | 4 ++-- roles/prereq_superset/meta/argument_specs.yml | 5 +++-- roles/prereq_supported/meta/argument_specs.yml | 1 + roles/prereq_thp/meta/argument_specs.yml | 5 +++-- roles/prereq_tls_acls/README.md | 2 +- roles/prereq_tls_acls/meta/argument_specs.yml | 7 ++++--- roles/prereq_yarn/meta/argument_specs.yml | 7 ++++--- roles/prereq_zeppelin/meta/argument_specs.yml | 7 ++++--- roles/prereq_zookeeper/meta/argument_specs.yml | 7 ++++--- 78 files changed, 210 insertions(+), 146 deletions(-) diff --git a/roles/prereq_accumulo/meta/argument_specs.yml b/roles/prereq_accumulo/meta/argument_specs.yml index 8d8a5e1c..0bf2dd49 100644 --- a/roles/prereq_accumulo/meta/argument_specs.yml +++ b/roles/prereq_accumulo/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Accumulo - description: | - Set up for Apache Accumulo usage, notably, create the local C(accumulo) user. + description: + - Set up for Apache Accumulo usage, notably, create the local C(accumulo) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_activitymonitor/meta/argument_specs.yml b/roles/prereq_activitymonitor/meta/argument_specs.yml index 9de4a75d..49a0ebfe 100644 --- a/roles/prereq_activitymonitor/meta/argument_specs.yml +++ b/roles/prereq_activitymonitor/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(activity_monitor_username), O(activity_monitor_password), and O(activity_monitor_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_atlas/meta/argument_specs.yml b/roles/prereq_atlas/meta/argument_specs.yml index 8699a424..7b6a3926 100644 --- a/roles/prereq_atlas/meta/argument_specs.yml +++ b/roles/prereq_atlas/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Atlas - description: | - Set up for Apache Atlas usage, notably, create the local C(atlas) user. + description: + - Set up for Apache Atlas usage, notably, create the local C(atlas) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_cloudera_manager/meta/argument_specs.yml b/roles/prereq_cloudera_manager/meta/argument_specs.yml index 61141b0f..b7f81e46 100644 --- a/roles/prereq_cloudera_manager/meta/argument_specs.yml +++ b/roles/prereq_cloudera_manager/meta/argument_specs.yml @@ -16,11 +16,12 @@ argument_specs: main: short_description: Set up user accounts and LDAP for Kerberos for Cloudera Manager - description: | - Set up for Cloudera Manager usage, notably, create the local C(cloudera-scm) user, including C(HOME) directory permissions. - Set up TLS ACLs, if needed. - Install LDAP packages for Kerberos, if needed. + description: + - Set up for Cloudera Manager usage, notably, create the local C(cloudera-scm) user, including C(HOME) directory permissions. + - Set up TLS ACLs, if needed. + - Install LDAP packages for Kerberos, if needed. author: Cloudera Labs + version_added: "5.0.0" options: kerberos_config_path: description: diff --git a/roles/prereq_cm_database/meta/argument_specs.yml b/roles/prereq_cm_database/meta/argument_specs.yml index 9a5fd62c..02f5bf1a 100644 --- a/roles/prereq_cm_database/meta/argument_specs.yml +++ b/roles/prereq_cm_database/meta/argument_specs.yml @@ -17,9 +17,10 @@ argument_specs: main: short_description: Set up database and user accounts for Cloudera Manager description: - - Creates the database and user for Cloudera Manager on PostgreSQL + - Creates the database and user for Cloudera Manager author: - "Jim Enright " + version_added: "5.0.0" options: cloudera_manager_database_user: description: Cloudera Manager database username diff --git a/roles/prereq_database/meta/argument_specs.yml b/roles/prereq_database/meta/argument_specs.yml index ba635858..47b7a583 100644 --- a/roles/prereq_database/meta/argument_specs.yml +++ b/roles/prereq_database/meta/argument_specs.yml @@ -16,9 +16,10 @@ argument_specs: main: short_description: Create and manage databases and users - description: | - This role configures databases and their associated users in the specified database system. + description: + - Configure databases and their associated users in the specified database system. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_dataviz/meta/argument_specs.yml b/roles/prereq_dataviz/meta/argument_specs.yml index b69dab7c..b3cb9474 100644 --- a/roles/prereq_dataviz/meta/argument_specs.yml +++ b/roles/prereq_dataviz/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Dataviz - description: | - Set up for Dataviz usage, notably, create the local C(dataviz) user. + description: + - Set up for Dataviz usage, notably, create the local C(dataviz) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_dataviz_database/meta/argument_specs.yml b/roles/prereq_dataviz_database/meta/argument_specs.yml index b7470cf8..cf5e74d4 100644 --- a/roles/prereq_dataviz_database/meta/argument_specs.yml +++ b/roles/prereq_dataviz_database/meta/argument_specs.yml @@ -18,9 +18,10 @@ argument_specs: short_description: Set up database and user accounts for Dataviz description: - Set up the Dataviz database and its associated user accounts, ensuring proper configuration for Dataviz operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the `dataviz_username`, - `dataviz_password`, and `dataviz_database` variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the c(dataviz_username), + C(dataviz_password), and C(dataviz_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_druid/meta/argument_specs.yml b/roles/prereq_druid/meta/argument_specs.yml index 8fd7fe4d..2cb4bf0f 100644 --- a/roles/prereq_druid/meta/argument_specs.yml +++ b/roles/prereq_druid/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Druid - description: | - Set up for Apache Druid usage, notably, create the local C(druid) user. + description: + - Set up for Apache Druid usage, notably, create the local C(druid) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_ecs/files/networkmanager.conf b/roles/prereq_ecs/files/networkmanager.conf index 13d2a834..16476e25 100644 --- a/roles/prereq_ecs/files/networkmanager.conf +++ b/roles/prereq_ecs/files/networkmanager.conf @@ -1,2 +1,2 @@ [keyfile] -unmanaged-devices=interface-name:cali*;interface-name:flannel* \ No newline at end of file +unmanaged-devices=interface-name:cali*;interface-name:flannel* diff --git a/roles/prereq_ecs/meta/argument_specs.yml b/roles/prereq_ecs/meta/argument_specs.yml index 2321af5f..1f705d89 100644 --- a/roles/prereq_ecs/meta/argument_specs.yml +++ b/roles/prereq_ecs/meta/argument_specs.yml @@ -21,4 +21,5 @@ argument_specs: of firewall and networking configuration. - The list of local users are taken from the P(cloudera.exe.prereq_cloudera_manager#role) role. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_firewall/meta/argument_specs.yml b/roles/prereq_firewall/meta/argument_specs.yml index 75346987..386ac5db 100644 --- a/roles/prereq_firewall/meta/argument_specs.yml +++ b/roles/prereq_firewall/meta/argument_specs.yml @@ -22,6 +22,7 @@ argument_specs: - Backup file naming is C(iptables-rules-[ipv4|ipv6].TIMESTAMP), where TIMESTAMP is UTC. author: - Webster Mudge + version_added: "5.0.0" options: firewall_backup_enabled: description: Flag to enable timestamped backups of iptable rules. diff --git a/roles/prereq_flink/meta/argument_specs.yml b/roles/prereq_flink/meta/argument_specs.yml index 010ca01e..2baf8d42 100644 --- a/roles/prereq_flink/meta/argument_specs.yml +++ b/roles/prereq_flink/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Flink - description: | - Set up for Apache Flink usage, notably, create the local C(flink) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Flink usage, notably, create the local C(flink) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_flume/meta/argument_specs.yml b/roles/prereq_flume/meta/argument_specs.yml index b4ef32f1..a546480d 100644 --- a/roles/prereq_flume/meta/argument_specs.yml +++ b/roles/prereq_flume/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Flume - description: | - Set up for Apache Flume usage, notably, create the local C(flume) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Flume usage, notably, create the local C(flume) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hadoop/meta/argument_specs.yml b/roles/prereq_hadoop/meta/argument_specs.yml index 9c9a6f1d..b2e48599 100644 --- a/roles/prereq_hadoop/meta/argument_specs.yml +++ b/roles/prereq_hadoop/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Hadoop - description: | - Set up for Apache Hadoop usage, notably, create the local C(hadoop) user group. + description: + - Set up for Apache Hadoop usage, notably, create the local C(hadoop) user group. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hbase/meta/argument_specs.yml b/roles/prereq_hbase/meta/argument_specs.yml index 730636a8..ed931b1c 100644 --- a/roles/prereq_hbase/meta/argument_specs.yml +++ b/roles/prereq_hbase/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for HBase - description: | - Set up for Apache HBase usage, notably, create the local C(hbase) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache HBase usage, notably, create the local C(hbase) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hdfs/meta/argument_specs.yml b/roles/prereq_hdfs/meta/argument_specs.yml index de674b9f..54523922 100644 --- a/roles/prereq_hdfs/meta/argument_specs.yml +++ b/roles/prereq_hdfs/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up for Hdfs - description: | - Set up for Hdfs usage, notably, create the local C(hdfs) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Hdfs usage, notably, create the local C(hdfs) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hive/meta/argument_specs.yml b/roles/prereq_hive/meta/argument_specs.yml index 2250557d..285bc9ee 100644 --- a/roles/prereq_hive/meta/argument_specs.yml +++ b/roles/prereq_hive/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Hive - description: | - Set up for Apache Hive usage, notably, create the local C(hive) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Hive usage, notably, create the local C(hive) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hive_database/meta/argument_specs.yml b/roles/prereq_hive_database/meta/argument_specs.yml index 3986cae1..dac23e7e 100644 --- a/roles/prereq_hive_database/meta/argument_specs.yml +++ b/roles/prereq_hive_database/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Default values are used for the username, password, and database name, but these can be overwritten by setting the `hive_username`, `hive_password`, and `hive_database` variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_httpfs/meta/argument_specs.yml b/roles/prereq_httpfs/meta/argument_specs.yml index 313dad0e..f60924b3 100644 --- a/roles/prereq_httpfs/meta/argument_specs.yml +++ b/roles/prereq_httpfs/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for HttpFS - description: | - Set up for HttpFS usage, notably, create the local C(httpfs) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for HttpFS usage, notably, create the local C(httpfs) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_hue/meta/argument_specs.yml b/roles/prereq_hue/meta/argument_specs.yml index b684ed4d..920284b5 100644 --- a/roles/prereq_hue/meta/argument_specs.yml +++ b/roles/prereq_hue/meta/argument_specs.yml @@ -16,11 +16,12 @@ argument_specs: main: short_description: Set up user accounts and Kerberos for Hue - description: | - Set up for Hue usage, notably, create the local C(hue) user. - Optionally, set up ACLs on TLS entities. - Optionally, update Kerberos encryption types. + description: + - Set up for Hue usage, notably, create the local C(hue) user. + - Optionally, set up ACLs on TLS entities. + - Optionally, update Kerberos encryption types. author: Cloudera Labs + version_added: "5.0.0" options: kerberos_config_path: description: diff --git a/roles/prereq_hue_database/meta/argument_specs.yml b/roles/prereq_hue_database/meta/argument_specs.yml index d9386db8..79724fbb 100644 --- a/roles/prereq_hue_database/meta/argument_specs.yml +++ b/roles/prereq_hue_database/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Default values are used for the username, password, and database name, but these can be overwritten by setting the `hue_username`, `hue_password`, and `hue_database` variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_impala/meta/argument_specs.yml b/roles/prereq_impala/meta/argument_specs.yml index c1c549cb..cd917c09 100644 --- a/roles/prereq_impala/meta/argument_specs.yml +++ b/roles/prereq_impala/meta/argument_specs.yml @@ -16,10 +16,11 @@ argument_specs: main: short_description: Set up user accounts for Impala - description: | - Set up for Impala usage, notably, create the local C(impala) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Impala usage, notably, create the local C(impala) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: tls_keystore_path: description: diff --git a/roles/prereq_jdk/library/jdk_facts.py b/roles/prereq_jdk/library/jdk_facts.py index 051a2780..5f9d0e4e 100644 --- a/roles/prereq_jdk/library/jdk_facts.py +++ b/roles/prereq_jdk/library/jdk_facts.py @@ -112,7 +112,7 @@ + "[+-_]?(?P[\\w\\d]*)" + "[+-_]?(?P[\\w\\d]*)" + ")" - + "\\)" + + "\\)", ) UPDATE_REGEX = re.compile('".+"\\s*([\\w-]*)') diff --git a/roles/prereq_jdk/meta/argument_specs.yml b/roles/prereq_jdk/meta/argument_specs.yml index feb96895..6cd976ea 100644 --- a/roles/prereq_jdk/meta/argument_specs.yml +++ b/roles/prereq_jdk/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - For JDK 9 and below, optionally enable the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy. - If the JDK is installed from the Cloudera repo, add any missing symlinks. author: Cloudera Labs + version_added: "5.0.0" options: jdk_provider: description: diff --git a/roles/prereq_kafka/meta/argument_specs.yml b/roles/prereq_kafka/meta/argument_specs.yml index e36e9826..ce5d168a 100644 --- a/roles/prereq_kafka/meta/argument_specs.yml +++ b/roles/prereq_kafka/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Kafka - description: | - Set up for Apache Kafka usage, notably, create the local C(kafka) and C(cruisecontrol) users. + description: + - Set up for Apache Kafka usage, notably, create the local C(kafka) and C(cruisecontrol) users. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_kerberos/meta/argument_specs.yml b/roles/prereq_kerberos/meta/argument_specs.yml index 956a5d44..c5373d46 100644 --- a/roles/prereq_kerberos/meta/argument_specs.yml +++ b/roles/prereq_kerberos/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Configure Kerberos credential cache. - Configure SSSD, if needed. author: Cloudera Labs + version_added: "5.0.0" options: kerberos_packages: description: diff --git a/roles/prereq_kernel/meta/argument_specs.yml b/roles/prereq_kernel/meta/argument_specs.yml index 46f48243..41e46a49 100644 --- a/roles/prereq_kernel/meta/argument_specs.yml +++ b/roles/prereq_kernel/meta/argument_specs.yml @@ -16,12 +16,13 @@ argument_specs: main: short_description: Update OS kernel parameters for deployments - description: | - Updates and applies kernel parameters using C(sysctl). - This includes controlling memory management (swappiness and memory overcommit) - and configuring IPv6 settings to disable IPv6 functionality. + description: + - Updates and applies kernel parameters using C(sysctl). + - This includes controlling memory management (swappiness and memory overcommit) + and configuring IPv6 settings to disable IPv6 functionality. author: - "Ronald Suplina " + version_added: "5.0.0" options: prereq_kernel__kernel_flags: description: Dictionary of kernel parameters to configure with C(sysctl). diff --git a/roles/prereq_keytrustee/meta/argument_specs.yml b/roles/prereq_keytrustee/meta/argument_specs.yml index e3c695bb..237b21b9 100644 --- a/roles/prereq_keytrustee/meta/argument_specs.yml +++ b/roles/prereq_keytrustee/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Key Trustee - description: | - Set up for Cloudera Navigator Key Trustee usage, notably, create the local C(keytrustee) users. + description: + - Set up for Cloudera Navigator Key Trustee usage, notably, create the local C(keytrustee) users. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_kms/meta/argument_specs.yml b/roles/prereq_kms/meta/argument_specs.yml index 5d8471ca..21cb629b 100644 --- a/roles/prereq_kms/meta/argument_specs.yml +++ b/roles/prereq_kms/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for KMS - description: | - Set up for Cloudera Key Management System (KMS) usage, notably, create the local C(kms) users. + description: + - Set up for Cloudera Key Management System (KMS) usage, notably, create the local C(kms) users. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_knox/meta/argument_specs.yml b/roles/prereq_knox/meta/argument_specs.yml index 22d30701..a0d7cab8 100644 --- a/roles/prereq_knox/meta/argument_specs.yml +++ b/roles/prereq_knox/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Knox - description: | - Set up for Apache Knox usage, notably, create the local C(knox) user. + description: + - Set up for Apache Knox usage, notably, create the local C(knox) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_knox_database/meta/argument_specs.yml b/roles/prereq_knox_database/meta/argument_specs.yml index 802ffb63..257f7c67 100644 --- a/roles/prereq_knox_database/meta/argument_specs.yml +++ b/roles/prereq_knox_database/meta/argument_specs.yml @@ -18,9 +18,10 @@ argument_specs: short_description: Set up database and user accounts for Knox description: - Set up the Apache Knox database and its associated user accounts, ensuring proper configuration for Knox operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the `knox_username`, `knox_password`, and - `knox_database` variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(knox_username), O(knox_password), and + O(knox_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_kudu/meta/argument_specs.yml b/roles/prereq_kudu/meta/argument_specs.yml index 35bfd5eb..f4c70660 100644 --- a/roles/prereq_kudu/meta/argument_specs.yml +++ b/roles/prereq_kudu/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Kudu - description: | - Set up for Apache Kudu usage, notably, create the local C(kudu) user. + description: + - Set up for Apache Kudu usage, notably, create the local C(kudu) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_livy/meta/argument_specs.yml b/roles/prereq_livy/meta/argument_specs.yml index a1ef4283..3f8cd2e7 100644 --- a/roles/prereq_livy/meta/argument_specs.yml +++ b/roles/prereq_livy/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Livy - description: | - Set up for Apache Livy usage, notably, create the local C(livy) user. + description: + - Set up for Apache Livy usage, notably, create the local C(livy) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_local_account/meta/argument_specs.yml b/roles/prereq_local_account/meta/argument_specs.yml index 25c699f0..1fc4052c 100644 --- a/roles/prereq_local_account/meta/argument_specs.yml +++ b/roles/prereq_local_account/meta/argument_specs.yml @@ -16,9 +16,10 @@ argument_specs: main: short_description: Set up local user accounts - description: | - Set up local user accounts and create the user's C(HOME) directory. + description: + - Set up local user accounts and create the user's C(HOME) directory. author: Cloudera Labs + version_added: "5.0.0" options: local_accounts: description: A list of user accounts. diff --git a/roles/prereq_mapreduce/meta/argument_specs.yml b/roles/prereq_mapreduce/meta/argument_specs.yml index d86248e0..68d5c4ec 100644 --- a/roles/prereq_mapreduce/meta/argument_specs.yml +++ b/roles/prereq_mapreduce/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for MapReduce - description: | - Set up for MapReduce usage, notably, create the local C(mapred) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for MapReduce usage, notably, create the local C(mapred) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_network_dns/meta/argument_specs.yml b/roles/prereq_network_dns/meta/argument_specs.yml index 989d0c00..e9dc3e37 100644 --- a/roles/prereq_network_dns/meta/argument_specs.yml +++ b/roles/prereq_network_dns/meta/argument_specs.yml @@ -16,10 +16,11 @@ argument_specs: main: short_description: Set up hostname and DNS networking - description: | - Set up hostname and DNS networking, targeting various systems that control such configuration, such as C(cloud-init) and C(NetworkManager). - Set up DNS forwarders for name resolution. + description: + - Set up hostname and DNS networking, targeting various systems that control such configuration, such as C(cloud-init) and C(NetworkManager). + - Set up DNS forwarders for name resolution. author: Cloudera Labs + version_added: "5.0.0" options: network_cloud_init_path: description: diff --git a/roles/prereq_nifi/meta/argument_specs.yml b/roles/prereq_nifi/meta/argument_specs.yml index 17a847f1..4470349d 100644 --- a/roles/prereq_nifi/meta/argument_specs.yml +++ b/roles/prereq_nifi/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for NiFi - description: | - Set up for Apache NiFi usage, notably, create the local C(nifi) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache NiFi usage, notably, create the local C(nifi) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_nifiregistry/meta/argument_specs.yml b/roles/prereq_nifiregistry/meta/argument_specs.yml index e19f3182..810661ac 100644 --- a/roles/prereq_nifiregistry/meta/argument_specs.yml +++ b/roles/prereq_nifiregistry/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for NiFi Registry - description: | - Set up for Apache NiFi Registry usage, notably, create the local C(nifiregistry) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache NiFi Registry usage, notably, create the local C(nifiregistry) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_ntp/meta/argument_specs.yml b/roles/prereq_ntp/meta/argument_specs.yml index 672e331d..760fe72c 100644 --- a/roles/prereq_ntp/meta/argument_specs.yml +++ b/roles/prereq_ntp/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up NTP services for deployments - description: | - This module installs the Chrony NTP package if neither Chrony nor NTP is currently installed or running on the system. - If both services are present and running, it will stop the NTP service to prioritize Chrony. + description: + - This module installs the Chrony NTP package if neither Chrony nor NTP is currently installed or running on the system. + - If both services are present and running, it will stop the NTP service to prioritize Chrony. author: "Ronald Suplina " + version_added: "5.0.0" options: {} diff --git a/roles/prereq_oozie/meta/argument_specs.yml b/roles/prereq_oozie/meta/argument_specs.yml index fbef2e24..8ab4526f 100644 --- a/roles/prereq_oozie/meta/argument_specs.yml +++ b/roles/prereq_oozie/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Oozie - description: | - Set up for Apache Oozie usage, notably, create the local C(oozie) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Oozie usage, notably, create the local C(oozie) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_oozie_database/meta/argument_specs.yml b/roles/prereq_oozie_database/meta/argument_specs.yml index 24360918..f80ca9dd 100644 --- a/roles/prereq_oozie_database/meta/argument_specs.yml +++ b/roles/prereq_oozie_database/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Default values are used for the username, password, and database name, but these can be overwritten by setting the `oozie_username`, `oozie_password`, and `oozie_database` variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_os/meta/argument_specs.yml b/roles/prereq_os/meta/argument_specs.yml index 512c3c17..e7bca196 100644 --- a/roles/prereq_os/meta/argument_specs.yml +++ b/roles/prereq_os/meta/argument_specs.yml @@ -22,6 +22,7 @@ argument_specs: - Update Ubuntu 20.04 root permissions to global C(/tmp) directory. author: - Webster Mudge (wmudge@cloudera.com) + version_added: "5.0.0" options: os_timezone: description: Timezone to set on the host. diff --git a/roles/prereq_phoenix/meta/argument_specs.yml b/roles/prereq_phoenix/meta/argument_specs.yml index 1e4119b9..1aa92a8c 100644 --- a/roles/prereq_phoenix/meta/argument_specs.yml +++ b/roles/prereq_phoenix/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Phoenix - description: | - Set up for Apache Phoenix usage, notably, create local C(phoenix) user. + description: + - Set up for Apache Phoenix usage, notably, create local C(phoenix) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_psycopg2/meta/argument_specs.yml b/roles/prereq_psycopg2/meta/argument_specs.yml index bc2d6889..b9839bf3 100644 --- a/roles/prereq_psycopg2/meta/argument_specs.yml +++ b/roles/prereq_psycopg2/meta/argument_specs.yml @@ -20,6 +20,7 @@ argument_specs: - Installs the psycopg2 Python package for PostgreSQL database. author: - "Jim Enright " + version_added: "5.0.0" options: rdbms_repo_disable: description: Disable PSQL repositories before installing psycopg2 diff --git a/roles/prereq_python/README.md b/roles/prereq_python/README.md index 6a21d770..11a49128 100644 --- a/roles/prereq_python/README.md +++ b/roles/prereq_python/README.md @@ -75,4 +75,4 @@ Copyright 2025 Cloudera, Inc. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` \ No newline at end of file +``` diff --git a/roles/prereq_python/meta/argument_specs.yml b/roles/prereq_python/meta/argument_specs.yml index b0c39f03..da6a6053 100644 --- a/roles/prereq_python/meta/argument_specs.yml +++ b/roles/prereq_python/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Also ensures that Pip is installed and updated. author: - "Jim Enright " + version_added: "5.0.0" options: cloudera_manager_version: description: Version of Cloudera Manager diff --git a/roles/prereq_query_processor_database/meta/argument_specs.yml b/roles/prereq_query_processor_database/meta/argument_specs.yml index 9307b78c..10fa95c5 100644 --- a/roles/prereq_query_processor_database/meta/argument_specs.yml +++ b/roles/prereq_query_processor_database/meta/argument_specs.yml @@ -18,9 +18,10 @@ argument_specs: short_description: Set up database and user accounts for Query Processor description: - Set up the Hue Query Processor database and its associated user accounts, ensuring proper configuration for Query Processor operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the `query_processor_username`, - `query_processor_password`, and `query_processor_database` variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(query_processor_username), + O(query_processor_password), and O(query_processor_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_ranger/meta/argument_specs.yml b/roles/prereq_ranger/meta/argument_specs.yml index 7e65ae8b..48cdb2e7 100644 --- a/roles/prereq_ranger/meta/argument_specs.yml +++ b/roles/prereq_ranger/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Ranger - description: | - Set up for Apache Ranger usage, notably, create the local C(ranger,rangerraz,rangertagsync) users and local C(ranger) group. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Ranger usage, notably, create the local C(ranger,rangerraz,rangertagsync) users and local C(ranger) group. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_ranger_database/meta/argument_specs.yml b/roles/prereq_ranger_database/meta/argument_specs.yml index 4bec0156..adf670ee 100644 --- a/roles/prereq_ranger_database/meta/argument_specs.yml +++ b/roles/prereq_ranger_database/meta/argument_specs.yml @@ -18,9 +18,10 @@ argument_specs: short_description: Set up database and user accounts for Ranger description: - Set up the Apache Ranger database and its associated user accounts, ensuring proper configuration for Ranger operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(ranger_username), - I(ranger_password), and I(ranger_database) variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(ranger_username), + O(ranger_password), and O(ranger_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_reportsmanager/meta/argument_specs.yml b/roles/prereq_reportsmanager/meta/argument_specs.yml index f17a101c..920b6b31 100644 --- a/roles/prereq_reportsmanager/meta/argument_specs.yml +++ b/roles/prereq_reportsmanager/meta/argument_specs.yml @@ -18,9 +18,10 @@ argument_specs: short_description: Set up database and user accounts for Reports Manager description: - Set up the Reports Manager database and its associated user accounts, ensuring proper configuration for Reports Manager operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the `reports_manager_username`, - `reports_manager_password`, and `reports_manager_database` variables. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(reports_manager_username), + O(reports_manager_password), and O(reports_manager_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_rngd/meta/argument_specs.yml b/roles/prereq_rngd/meta/argument_specs.yml index 93442c5b..74c90eee 100644 --- a/roles/prereq_rngd/meta/argument_specs.yml +++ b/roles/prereq_rngd/meta/argument_specs.yml @@ -20,4 +20,5 @@ argument_specs: - Installs and configures the Random Number Generator (rngd) package. author: - "Jim Enright " + version_added: "5.0.0" options: {} diff --git a/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 b/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 index 19e1f3dc..40551b34 100644 --- a/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 +++ b/roles/prereq_rngd/templates/rngd_Ubuntu.service.j2 @@ -22,4 +22,4 @@ EnvironmentFile=-/etc/default/rng-tools ExecStart=/usr/sbin/rngd -f -r $HRNGDEVICE $RNGDOPTIONS [Install] -WantedBy=dev-hwrng.device \ No newline at end of file +WantedBy=dev-hwrng.device diff --git a/roles/prereq_schemaregistry/meta/argument_specs.yml b/roles/prereq_schemaregistry/meta/argument_specs.yml index 0ff1a362..10eaca96 100644 --- a/roles/prereq_schemaregistry/meta/argument_specs.yml +++ b/roles/prereq_schemaregistry/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Schema Registry - description: | - Set up for Schema Registry usage, notably, create the local C(schemaregistry) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Schema Registry usage, notably, create the local C(schemaregistry) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_schemaregistry_database/meta/argument_specs.yml b/roles/prereq_schemaregistry_database/meta/argument_specs.yml index 8e670732..6c186dbf 100644 --- a/roles/prereq_schemaregistry_database/meta/argument_specs.yml +++ b/roles/prereq_schemaregistry_database/meta/argument_specs.yml @@ -16,9 +16,10 @@ argument_specs: main: short_description: Set up database and user accounts for Schema Registry - description: | - This role manages the creation of the database and its associated user accounts for Schema Registry. + description: + - Manage the creation of the database and its associated user accounts for Schema Registry. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_selinux/meta/argument_specs.yml b/roles/prereq_selinux/meta/argument_specs.yml index 650d227c..01158416 100644 --- a/roles/prereq_selinux/meta/argument_specs.yml +++ b/roles/prereq_selinux/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Installs packages for SELinux management, if needed, but does not install SELinux. author: - Webster Mudge (wmudge@cloudera.com) + version_added: "5.0.0" options: selinux_state: description: diff --git a/roles/prereq_sentry/meta/argument_specs.yml b/roles/prereq_sentry/meta/argument_specs.yml index 31b62688..c50625ad 100644 --- a/roles/prereq_sentry/meta/argument_specs.yml +++ b/roles/prereq_sentry/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Sentry - description: | - Set up for Apache Sentry usage, notably, create the local C(sentry) user. + description: + - Set up for Apache Sentry usage, notably, create the local C(sentry) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_services/meta/argument_specs.yml b/roles/prereq_services/meta/argument_specs.yml index a01d41a3..325882ea 100644 --- a/roles/prereq_services/meta/argument_specs.yml +++ b/roles/prereq_services/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Includes installing and configuring required services as well as disabling unnecessary services. author: - "Jim Enright " + version_added: "5.0.0" options: prereq_services_unnecessary_services: description: "List of unwanted OS services that will be disabled." diff --git a/roles/prereq_smm/meta/argument_specs.yml b/roles/prereq_smm/meta/argument_specs.yml index 39138318..849c6d62 100644 --- a/roles/prereq_smm/meta/argument_specs.yml +++ b/roles/prereq_smm/meta/argument_specs.yml @@ -20,4 +20,5 @@ argument_specs: - Set up for Streams Messaging Manager usage, notably, create the local C(streamsmsgmgr) and C(streamsrepmgr) users. - Also creates a symbolic link of the Streams Messaging Manager user home directory required for the Custom Service Descriptor (CSD) installation. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_smm_database/meta/argument_specs.yml b/roles/prereq_smm_database/meta/argument_specs.yml index 274e8626..b8e9df84 100644 --- a/roles/prereq_smm_database/meta/argument_specs.yml +++ b/roles/prereq_smm_database/meta/argument_specs.yml @@ -14,11 +14,13 @@ argument_specs: main: - short_description: Set up database and user accounts for Streams Messaging Manager + short_description: Set up database and user accounts for Streams Messaging Manager description: - - Set up the Streams Messaging Manager database and its associated user accounts, ensuring proper configuration for Streams Messaging Manager operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(smm_username), I(smm_password), and I(smm_database) variables. + - Set up the Streams Messaging Manager database and its associated user accounts, ensuring proper configuration for Streams Messaging Manager operations. + - Default values are used for the username, password, and database name, but these can be overwritten by setting the O(smm_username), O(smm_password), + and I(smm_database) variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_smm_database/molecule/default/prepare.yml b/roles/prereq_smm_database/molecule/default/prepare.yml index 2b8546dd..8950648c 100644 --- a/roles/prereq_smm_database/molecule/default/prepare.yml +++ b/roles/prereq_smm_database/molecule/default/prepare.yml @@ -30,4 +30,3 @@ name: "{{ database_admin_user }}" password: "{{ database_admin_password }}" become_user: postgres - \ No newline at end of file diff --git a/roles/prereq_smm_database/molecule/default/requirements.yml b/roles/prereq_smm_database/molecule/default/requirements.yml index cd0f1849..3ce89d9d 100644 --- a/roles/prereq_smm_database/molecule/default/requirements.yml +++ b/roles/prereq_smm_database/molecule/default/requirements.yml @@ -19,4 +19,3 @@ collections: - ansible.utils - community.general - community.postgresql - diff --git a/roles/prereq_smm_database/molecule/default/verify.yml b/roles/prereq_smm_database/molecule/default/verify.yml index 2703afec..08c243ef 100644 --- a/roles/prereq_smm_database/molecule/default/verify.yml +++ b/roles/prereq_smm_database/molecule/default/verify.yml @@ -46,4 +46,3 @@ ansible.builtin.fail: msg: "User streamsmsgmgr does not exist!" when: user_check.query_result | length == 0 - diff --git a/roles/prereq_smm_database/tasks/main.yml b/roles/prereq_smm_database/tasks/main.yml index 21b6e01f..c6d8d72e 100644 --- a/roles/prereq_smm_database/tasks/main.yml +++ b/roles/prereq_smm_database/tasks/main.yml @@ -23,5 +23,3 @@ password: "{{ smm_password }}" db: "{{ smm_database }}" no_log: true - - diff --git a/roles/prereq_solr/meta/argument_specs.yml b/roles/prereq_solr/meta/argument_specs.yml index ae8e1335..bdba36ca 100644 --- a/roles/prereq_solr/meta/argument_specs.yml +++ b/roles/prereq_solr/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Solr - description: | - Set up for Apache Solr usage, notably, create the local C(solr) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Solr usage, notably, create the local C(solr) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_spark/meta/argument_specs.yml b/roles/prereq_spark/meta/argument_specs.yml index cde65253..92c96b19 100644 --- a/roles/prereq_spark/meta/argument_specs.yml +++ b/roles/prereq_spark/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Spark - description: | - Set up for Apache Spark usage, notably, create the local C(spark) user and local C(spark) group. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Spark usage, notably, create the local C(spark) user and local C(spark) group. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_spark2/meta/argument_specs.yml b/roles/prereq_spark2/meta/argument_specs.yml index 66aa981c..0c403304 100644 --- a/roles/prereq_spark2/meta/argument_specs.yml +++ b/roles/prereq_spark2/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Spark2 - description: | - Set up for Apache Spark2 usage, notably, create the local C(spark2) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Spark2 usage, notably, create the local C(spark2) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_sqoop/meta/argument_specs.yml b/roles/prereq_sqoop/meta/argument_specs.yml index 2bde7e19..0e450db9 100644 --- a/roles/prereq_sqoop/meta/argument_specs.yml +++ b/roles/prereq_sqoop/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Sqoop - description: | - Set up for Apache Sqoop usage, notably, create the local C(sqoop,sqoop2) users and local C(sqoop) group. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Sqoop usage, notably, create the local C(sqoop,sqoop2) users and local C(sqoop) group. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_ssb/meta/argument_specs.yml b/roles/prereq_ssb/meta/argument_specs.yml index 5e17d926..ce37e735 100644 --- a/roles/prereq_ssb/meta/argument_specs.yml +++ b/roles/prereq_ssb/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for SSB - description: | - Set up for SQL Stream Builder (SSB) usage, notably, create the local C(ssb) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for SQL Stream Builder (SSB) usage, notably, create the local C(ssb) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_ssb_database/meta/argument_specs.yml b/roles/prereq_ssb_database/meta/argument_specs.yml index ab06459f..ca9bcb2f 100644 --- a/roles/prereq_ssb_database/meta/argument_specs.yml +++ b/roles/prereq_ssb_database/meta/argument_specs.yml @@ -18,9 +18,9 @@ argument_specs: short_description: Set up database and user accounts for SQL Stream Builder description: - Set up the SQL Stream Builder (SSB) database and its associated user accounts, ensuring proper configuration for SSB operations. - - Default values are used for the username, password, and database name, but these can be overwritten by setting the I(ssb_username), I(ssb_password), and - I(ssb_database) variables. + - Default values are used for the username, password, and database names, but these can be overwritten by setting appropriate variables. author: Cloudera Labs + version_added: "5.0.0" options: database_type: description: Specifies the type of database to connect to (e.g., PostgreSQL, MySQL, Oracle). diff --git a/roles/prereq_superset/meta/argument_specs.yml b/roles/prereq_superset/meta/argument_specs.yml index 36f2323a..87475fd3 100644 --- a/roles/prereq_superset/meta/argument_specs.yml +++ b/roles/prereq_superset/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Set up user accounts for Superset - description: | - Set up for Apache Superset usage, notably, create the local C(superset) user. + description: + - Set up for Apache Superset usage, notably, create the local C(superset) user. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_supported/meta/argument_specs.yml b/roles/prereq_supported/meta/argument_specs.yml index 8da5063a..91d0d847 100644 --- a/roles/prereq_supported/meta/argument_specs.yml +++ b/roles/prereq_supported/meta/argument_specs.yml @@ -21,6 +21,7 @@ argument_specs: - Additionally, the I(support_matrix) variable defined in this role can be imported and used in other roles. author: - "Jim Enright " + version_added: "5.0.0" options: cloudera_manager_version: description: Version of Cloudera Manager diff --git a/roles/prereq_thp/meta/argument_specs.yml b/roles/prereq_thp/meta/argument_specs.yml index 5b6ac89a..5d887185 100644 --- a/roles/prereq_thp/meta/argument_specs.yml +++ b/roles/prereq_thp/meta/argument_specs.yml @@ -16,7 +16,8 @@ argument_specs: main: short_description: Disable Transparent Huge Pages for deployments - description: | - Disable Transparent Huge Pages (THP) and, optionally, rebuild the GRUB bootloader. + description: + - Disable Transparent Huge Pages (THP) and, optionally, rebuild the GRUB bootloader. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_tls_acls/README.md b/roles/prereq_tls_acls/README.md index 8f1e17f1..b18e221f 100644 --- a/roles/prereq_tls_acls/README.md +++ b/roles/prereq_tls_acls/README.md @@ -9,7 +9,7 @@ Typically, TLS entity variables are set as `hostvars`. The role will: - **`main` mode**: - Iterate through the `acl_user_accounts` list. - - For each specified user, it will apply `read` file ACLs for the users' `group` to the TLS keystore, encrypted private key, unencrypted private key, and password file, as directed by the corresponding Boolean flags (`keystore_acl`, `key_acl`, etc.). + - For each specified user, it will apply `read` file ACLs for the users' `group` to the TLS keystore, encrypted private key, unencrypted private key, and password file, as directed by the corresponding Boolean flags (`keystore_acl`, `key_acl`, etc.). - It uses the specified TLS path variables to locate the files to which ACLs should be applied. - **`validate` mode**: - Iterate through the `acl_user_accounts` list. diff --git a/roles/prereq_tls_acls/meta/argument_specs.yml b/roles/prereq_tls_acls/meta/argument_specs.yml index aebc533d..5eefc169 100644 --- a/roles/prereq_tls_acls/meta/argument_specs.yml +++ b/roles/prereq_tls_acls/meta/argument_specs.yml @@ -16,10 +16,11 @@ argument_specs: main: short_description: Set up local user ACLs for TLS - description: | - Set up local user ACLs for TLS entities, i.e. TLS keystore, private key, and password file. - The TLS entity variables are typically set as C(hostvars). + description: + - Set up local user ACLs for TLS entities, i.e. TLS keystore, private key, and password file. + - The TLS entity variables are typically set as C(hostvars). author: Cloudera Labs + version_added: "5.0.0" options: acl_user_accounts: description: A list of user accounts to apply to the TLS entities. diff --git a/roles/prereq_yarn/meta/argument_specs.yml b/roles/prereq_yarn/meta/argument_specs.yml index 72f67fff..312f8076 100644 --- a/roles/prereq_yarn/meta/argument_specs.yml +++ b/roles/prereq_yarn/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for YARN - description: | - Set up for Apache Hadoop YARN usage, notably, create the local C(yarn) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Hadoop YARN usage, notably, create the local C(yarn) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_zeppelin/meta/argument_specs.yml b/roles/prereq_zeppelin/meta/argument_specs.yml index 97505359..403388f9 100644 --- a/roles/prereq_zeppelin/meta/argument_specs.yml +++ b/roles/prereq_zeppelin/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up user accounts for Zeppelin - description: | - Set up for Apache Zeppelin usage, notably, create the local C(zeppelin) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Zeppelin usage, notably, create the local C(zeppelin) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {} diff --git a/roles/prereq_zookeeper/meta/argument_specs.yml b/roles/prereq_zookeeper/meta/argument_specs.yml index 5f1ccbe4..cf00e23b 100644 --- a/roles/prereq_zookeeper/meta/argument_specs.yml +++ b/roles/prereq_zookeeper/meta/argument_specs.yml @@ -16,8 +16,9 @@ argument_specs: main: short_description: Set up for Zookeeper - description: | - Set up for Apache Zookeeper usage, notably, create the local C(zookeeper) user. - Optionally, set up ACLs on TLS entities. + description: + - Set up for Apache Zookeeper usage, notably, create the local C(zookeeper) user. + - Optionally, set up ACLs on TLS entities. author: Cloudera Labs + version_added: "5.0.0" options: {}