Fixed handling of custom roleConfig Groups (#46)

* Fixed handling of custom roleConfigGroups.

When a roleConfigGroup was referenced in multiple templates it would be
duplicated in the final cluster template. This fix removes the duplication.

Signed-off-by: Andre Araujo <[email protected]>

* Removed deprecated SMM parameters from template.

Signed-off-by: Andre Araujo <[email protected]>

* Made error message more informative.

Signed-off-by: Andre Araujo <[email protected]>

Author: asdaraujo

plugins/filter/filters.py

@@ -211,17 +211,17 @@ def extract_role_and_group(self, role_spec):
         return (role, template_group)
 
     def extract_custom_roles(self, host_templates, service):
-        custom_roles = []
+        custom_roles = set([])
         for role_mapping in host_templates.values():
             if service in role_mapping:
                 for custom_role in filter(lambda x: '/' in x, role_mapping[service]):
-                    custom_roles.append(custom_role)
-        return custom_roles
+                    custom_roles.add(custom_role)
+        return list(custom_roles)
 
     def extract_custom_role_groups(self, host_templates):
-        custom_role_groups = []
+        custom_role_groups = set([])
         for role_mapping in host_templates.values():
             for (service, roles) in role_mapping.items():
                 for custom_role in filter(lambda x: '/' in x, roles):
-                    custom_role_groups.append("-".join([service.lower()] + custom_role.split("/")))
-        return custom_role_groups
+                    custom_role_groups.add("-".join([service.lower()] + custom_role.split("/")))
+        return list(custom_role_groups)

roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2

@@ -30,6 +30,3 @@ SPARK3_ON_YARN:
   GATEWAY:
     spark_io_encryption_enabled: true
     spark_network_encryption_enabled: true
-STREAMS_MESSAGING_MANAGER:
-  STREAMS_MESSAGING_MANAGER_SERVER:
-    streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.tls | default(false) | ternary('SASL_SSL', 'SASL_PLAINTEXT') }}

roles/config/cluster/base/templates/configs/tls-cm-7.j2

@@ -201,7 +201,6 @@ STREAMS_MESSAGING_MANAGER:
     ssl_server_keystore_keypassword: {{ tls_keystore_password }}
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
-    streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.kerberos | default(false) | ternary('SASL_SSL', 'SSL') }}
   STREAMS_MESSAGING_MANAGER_UI:
     ssl_client_truststore_location: {{ tls_chain_path }}
     ssl_enabled: true

roles/verify/parcels_and_roles/tasks/check_template_roles.yml

@@ -32,6 +32,7 @@
         fail_msg: >-
           Unknown role(s) {{ invalid_roles }} for service '{{ template.service }}'
           defined in host template '{{ host_template.name }}'.
+          Valid roles are: {{ role_mappings[template.service] | list }}.
 
     - name: Ensure the Tez gateway has been deployed
       assert: