From 580e027045878f8a8d97c0ad40f7ec9d80555b50 Mon Sep 17 00:00:00 2001 From: Andre Araujo Date: Wed, 2 Mar 2022 16:07:38 +1100 Subject: [PATCH 1/3] Fixed handling of custom roleConfigGroups. When a roleConfigGroup was referenced in multiple templates it would be duplicated in the final cluster template. This fix removes the duplication. Signed-off-by: Andre Araujo --- plugins/filter/filters.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/filter/filters.py b/plugins/filter/filters.py index 85263cc7..3db27222 100644 --- a/plugins/filter/filters.py +++ b/plugins/filter/filters.py @@ -211,17 +211,17 @@ def extract_role_and_group(self, role_spec): return (role, template_group) def extract_custom_roles(self, host_templates, service): - custom_roles = [] + custom_roles = set([]) for role_mapping in host_templates.values(): if service in role_mapping: for custom_role in filter(lambda x: '/' in x, role_mapping[service]): - custom_roles.append(custom_role) - return custom_roles + custom_roles.add(custom_role) + return list(custom_roles) def extract_custom_role_groups(self, host_templates): - custom_role_groups = [] + custom_role_groups = set([]) for role_mapping in host_templates.values(): for (service, roles) in role_mapping.items(): for custom_role in filter(lambda x: '/' in x, roles): - custom_role_groups.append("-".join([service.lower()] + custom_role.split("/"))) - return custom_role_groups + custom_role_groups.add("-".join([service.lower()] + custom_role.split("/"))) + return list(custom_role_groups) From f382c543a2323349ffd36faaa6f5dc1e847977e7 Mon Sep 17 00:00:00 2001 From: Andre Araujo Date: Wed, 2 Mar 2022 16:10:09 +1100 Subject: [PATCH 2/3] Removed deprecated SMM parameters from template. Signed-off-by: Andre Araujo --- .../config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 | 3 --- roles/config/cluster/base/templates/configs/tls-cm-7.j2 | 1 - 2 files changed, 4 deletions(-) diff --git a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 index 961f616a..b7dd4b11 100644 --- a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 +++ b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 @@ -30,6 +30,3 @@ SPARK3_ON_YARN: GATEWAY: spark_io_encryption_enabled: true spark_network_encryption_enabled: true -STREAMS_MESSAGING_MANAGER: - STREAMS_MESSAGING_MANAGER_SERVER: - streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.tls | default(false) | ternary('SASL_SSL', 'SASL_PLAINTEXT') }} diff --git a/roles/config/cluster/base/templates/configs/tls-cm-7.j2 b/roles/config/cluster/base/templates/configs/tls-cm-7.j2 index a381b33f..4d3aabce 100644 --- a/roles/config/cluster/base/templates/configs/tls-cm-7.j2 +++ b/roles/config/cluster/base/templates/configs/tls-cm-7.j2 @@ -201,7 +201,6 @@ STREAMS_MESSAGING_MANAGER: ssl_server_keystore_keypassword: {{ tls_keystore_password }} ssl_server_keystore_location: {{ tls_keystore_path_generic }} ssl_server_keystore_password: {{ tls_keystore_password }} - streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.kerberos | default(false) | ternary('SASL_SSL', 'SSL') }} STREAMS_MESSAGING_MANAGER_UI: ssl_client_truststore_location: {{ tls_chain_path }} ssl_enabled: true From 16222e1ea7477fa9d6909403c7f2ca9ea319ccc1 Mon Sep 17 00:00:00 2001 From: Andre Araujo Date: Wed, 2 Mar 2022 16:20:46 +1100 Subject: [PATCH 3/3] Made error message more informative. Signed-off-by: Andre Araujo --- roles/verify/parcels_and_roles/tasks/check_template_roles.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/verify/parcels_and_roles/tasks/check_template_roles.yml b/roles/verify/parcels_and_roles/tasks/check_template_roles.yml index 2efc625c..9b1cd338 100644 --- a/roles/verify/parcels_and_roles/tasks/check_template_roles.yml +++ b/roles/verify/parcels_and_roles/tasks/check_template_roles.yml @@ -32,6 +32,7 @@ fail_msg: >- Unknown role(s) {{ invalid_roles }} for service '{{ template.service }}' defined in host template '{{ host_template.name }}'. + Valid roles are: {{ role_mappings[template.service] | list }}. - name: Ensure the Tez gateway has been deployed assert: