RAZ Implementation for Azure

docs/configuration.yml

@@ -189,13 +189,15 @@ env:
         identity:
         log:
         ranger_audit:
+        raz: 
         xaccount:
       name:
         cross_account:
         datalake_admin:
         idbroker:
         log:
         ranger_audit:
+        raz: 
       name_suffix:
         admin:
         assignment:
@@ -240,6 +242,7 @@ env:
     user_sync:
     version:
     scale:
+    enable_raz: 
   gcp:
     auto_enable_services:
     manage_identities:
@@ -320,6 +323,7 @@ globals:
     private:
     public:
     ranger_audit:
+    raz:
     role:
     service_network:
     storage:

roles/common/defaults/main.yml

@@ -45,6 +45,7 @@ common__data_suffix:                      "{{ globals.labels.data | default('dat
 common__external_data_suffix:             "{{ globals.labels.external_data | default('external') }}"
 common__datalake_admin_suffix:            "{{ globals.labels.datalake_admin | default('dladmin') }}"
 common__ranger_audit_suffix:              "{{ globals.labels.ranger_audit | default('audit') }}"
+common__raz_suffix:                       "{{ globals.labels.raz | default('raz') }}"
 common__cml_suffix:                       "{{ globals.labels.cml | default('cml') }}"
 common__cde_suffix:                       "{{ globals.labels.cde | default('cde') }}"
 common__igw_suffix:                       "{{ globals.labels.internet_gateway | default('igw') }}"

roles/platform/defaults/main.yml

@@ -188,6 +188,7 @@ plat__azure_data_suffix:                      "{{ env.azure.role.label.data | de
 plat__azure_identity_suffix:                  "{{ env.azure.role.label.identity | default(common__identity_suffix) }}"
 plat__azure_datalake_admin_suffix:            "{{ env.azure.role.label.datalake_admin | default(common__datalake_admin_suffix) }}"
 plat__azure_ranger_audit_suffix:              "{{ env.azure.role.label.ranger_audit | default(common__ranger_audit_suffix) }}"
+plat__azure_raz_suffix:                       "{{ env.azure.role.label.raz | default(common__raz_suffix) }}"
 plat__azure_idbroker_suffix:                  "{{ env.azure.role.label.idbroker | default(common__idbroker_suffix) }}"
 
 plat__azure_owner_name_suffix:                "{{ env.azure.role.name_suffix.owner | default('owner') }}"
@@ -211,6 +212,7 @@ plat__azure_policy_url:                       "{{ env.azure.policy.url | default
 plat__azure_log_identity_name:                "{{ env.azure.role.name.log | default([plat__namespace, plat__azure_log_suffix, plat__azure_identity_suffix] | join('-')) }}"
 plat__azure_datalakeadmin_identity_name:      "{{ env.azure.role.name.datalake_admin | default([plat__namespace, plat__azure_datalake_admin_suffix, plat__azure_identity_suffix] | join('-')) }}"
 plat__azure_ranger_audit_identity_name:       "{{ env.azure.role.name.ranger_audit | default([plat__namespace, plat__azure_ranger_audit_suffix, plat__azure_identity_suffix] | join('-')) }}"
+plat__azure_raz_identity_name:                "{{ env.azure.role.name.raz | default([plat__namespace, plat__azure_raz_suffix, plat__azure_identity_suffix] | join('-')) }}"
 plat__azure_idbroker_identity_name:           "{{ env.azure.role.name.idbroker | default([plat__namespace, plat__azure_idbroker_suffix, plat__azure_identity_suffix] | join('-')) }}"
 
 plat__azure_xaccount_contributor_assn_name:             "{{ env.azure.role.assignment.cross_account.contributor | default('-'.join([plat__namespace, plat__azure_xaccount_suffix, plat__azure_contributor_name_suffix, plat__azure_assignment_name_suffix,ansible_date_time.iso8601]) | to_uuid )}}"

roles/platform/tasks/initialize_azure.yml

@@ -144,4 +144,20 @@
   ansible.builtin.set_fact:
     plat__azure_storage_account_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}"
     plat__azure_logpath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ plat__azure_log_suffix }}"
-    plat__azure_datapath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ plat__azure_data_suffix }}"
+    plat__azure_datapath_uri: "{{ plat__azure_metagroup_uri }}/providers/Microsoft.Storage/storageAccounts/{{ plat__azure_storage_name }}/blobServices/default/containers/{{ plat__azure_data_suffix }}"
+
+- name: Set fact for default Azure MSIs
+  ansible.builtin.set_fact:
+    plat__azure_msis: "{{ plat__azure_msis | default([]) | union([__azure_msi_item]) }}"
+  loop_control:
+    loop_var: __azure_msi_item
+  loop: 
+    - "{{ plat__azure_idbroker_identity_name }}"
+    - "{{ plat__azure_datalakeadmin_identity_name }}"
+    - "{{ plat__azure_log_identity_name }}"
+    - "{{ plat__azure_ranger_audit_identity_name }}"
+
+- name: Update fact for RAZ Azure MSI
+  when: plat__enable_raz | bool
+  ansible.builtin.set_fact:
+    plat__azure_msis: "{{ plat__azure_msis | union([plat__azure_raz_identity_name]) }}"

roles/platform/tasks/initialize_teardown_azure.yml

@@ -40,5 +40,5 @@
       - "{{ plat__azure_datalakeadmin_identity_name }}"
       - "{{ plat__azure_log_identity_name }}"
       - "{{ plat__azure_ranger_audit_identity_name }}"
-    role_assignment_assignee_list: "{{ msi_principal_id_list | union([plat__azure_application_service_principal_objuuid | default([])]) | list }}"
+    role_assignment_assignee_list: "{{ msi_principal_id_list | union([plat__azure_application_service_principal_objuuid] if plat__azure_application_service_principal_objuuid is defined else []) }}"
 

roles/platform/tasks/setup_azure_authz.yml

@@ -16,6 +16,7 @@
 
 - name: Handle Azure Cross Account Role
   register: __azure_xaccount_role_info
+  when: plat__azure_xaccount_use_custom_role | bool
   azure.azcollection.azure_rm_roledefinition:  # This version fails idempotence if a description is set
     state: present
     name: "{{ plat__azure_xaccount_role_name }}"
@@ -121,11 +122,7 @@
     state: present
     body:
       location: "{{ plat__region }}"
-  loop:
-    - "{{ plat__azure_idbroker_identity_name }}"
-    - "{{ plat__azure_datalakeadmin_identity_name }}"
-    - "{{ plat__azure_log_identity_name }}"
-    - "{{ plat__azure_ranger_audit_identity_name }}"
+  loop: "{{ plat__azure_msis }}"
 
 - name: Wait for MSIs to be listed
   azure.azcollection.azure_rm_resource_info:
@@ -136,11 +133,7 @@
   register: __azure_identity_list
   delay: 5
   retries: 120 # 10 mins
-  until:
-    - plat__azure_idbroker_identity_name in discovered_msi_list
-    - plat__azure_datalakeadmin_identity_name in discovered_msi_list
-    - plat__azure_log_identity_name in discovered_msi_list
-    - plat__azure_ranger_audit_identity_name in discovered_msi_list
+  until: discovered_msi_list is subset(plat__azure_msis)
   vars:
     discovered_msi_list: "{{ __azure_identity_list.response | map(attribute='name') | list }}"
 
@@ -150,6 +143,7 @@
     __azure_datalakeadmin_identity: "{{ __azure_identity_list.response | selectattr('name', 'eq', plat__azure_datalakeadmin_identity_name) | first }}"
     __azure_log_identity: "{{ __azure_identity_list.response | selectattr('name', 'eq', plat__azure_log_identity_name) | first }}"
     __azure_ranger_audit_identity: "{{ __azure_identity_list.response | selectattr('name', 'eq', plat__azure_ranger_audit_identity_name) | first }}"
+    __azure_raz_identity: "{{ __azure_identity_list.response | selectattr('name', 'eq', plat__azure_raz_identity_name) | first | default(omit) }}"
 
 - name: Refresh listing of Azure Role Definitions
   register: __azure_role_definition_info
@@ -159,12 +153,12 @@
 
 - name: Extract Azure Role Definition IDs
   ansible.builtin.set_fact:
-    __azure_xaccount_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_xaccount_role_name) | map(attribute='id') | list | first }}"
     __azure_virtualmachine_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.vmcnt) | map(attribute='id') | list | first }}"
     __azure_managedidentity_optr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.miop) | map(attribute='id') | list | first }}"
     __azure_storageblobdata_ownr_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storown) | map(attribute='id') | list | first }}"
     __azure_storageblobdata_ctrb_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.storcnt) | map(attribute='id') | list | first }}"
     __azure_contributor_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.contrib) | map(attribute='id') | list | first }}"
+    __azure_storageblob_delegator_role_id: "{{ __azure_role_definition_info.roledefinitions | selectattr('role_name', 'eq', plat__azure_roles.stordel) | map(attribute='id') | list | first }}"
 
 - name: Process Azure Role Assignments
   register: __infra_az_sp_assign_result
@@ -213,3 +207,31 @@
     - "'rc' in __infra_az_sp_assign_result" 
     - __infra_az_sp_assign_result.rc != 0
     - "'The role assignment already exists' not in __infra_az_sp_assign_result.module_stderr"
+
+- name: Process RAZ Azure Role Assignments
+  register: __infra_az_sp_assign_raz_result
+  until: __infra_az_sp_assign_raz_result is not failed
+  when: plat__enable_raz | bool
+  retries: 3
+  delay: 3
+  azure.azcollection.azure_rm_roleassignment:
+    state: present
+    scope: "{{ __azure_rl_assgn_raz_item.scope }}"
+    assignee_object_id: "{{ __azure_rl_assgn_raz_item.assignee }}"
+    role_definition_id: "{{ __azure_rl_assgn_raz_item.role }}"
+  loop:
+    - role: "{{ __azure_storageblob_delegator_role_id }}"
+      scope: "{{ plat__azure_storage_account_uri }}"
+      assignee: "{{ __azure_raz_identity.properties.principalId }}"
+      desc: Assign Storage Blob Delegator Role to RAZ Identity at Storage Account level
+    - role: "{{ __azure_storageblobdata_ownr_role_id }}"
+      scope: "{{ plat__azure_storage_account_uri }}"
+      assignee: "{{ __azure_raz_identity.properties.principalId }}"
+      desc: Assign Storage Blob Data Owner Role to RAZ Identity at Storage Account level
+  loop_control:
+    loop_var: __azure_rl_assgn_raz_item
+    label: "{{ __azure_rl_assgn_raz_item.desc }}"
+  failed_when:
+    - "'rc' in __infra_az_sp_assign_raz_result" 
+    - __infra_az_sp_assign_raz_result.rc != 0
+    - "'The role assignment already exists' not in __infra_az_sp_assign_raz_result.module_stderr"

roles/platform/tasks/setup_azure_datalake.yml

@@ -23,4 +23,5 @@
     runtime: "{{ plat__datalake_version | default(omit) }}"
     scale: "{{ plat__datalake_scale | default(omit) }}"
     tags: "{{ plat__tags }}"
+    raz: "{{ plat__enable_raz }}"
     state: present

roles/platform/tasks/setup_azure_idbroker.yml

@@ -20,6 +20,7 @@
     sync: no
     data_access: "{{ __azure_datalakeadmin_identity.id }}"
     ranger_audit: "{{ __azure_ranger_audit_identity.id }}"
+    ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(__azure_raz_identity.id, omit) }}"
     mappings:
       - accessor: "{{ plat__cdp_pub_admin_group_crn }}"
         role: "{{ __azure_datalakeadmin_identity.id }}"

roles/platform/tasks/teardown_azure_authz.yml

@@ -39,13 +39,10 @@
     api_version: '2018-11-30'
     idempotency: yes
     state: absent
-  loop:
-    - "{{ plat__azure_idbroker_identity_name }}"
-    - "{{ plat__azure_datalakeadmin_identity_name }}"
-    - "{{ plat__azure_log_identity_name }}"
-    - "{{ plat__azure_ranger_audit_identity_name }}"
+  loop: "{{ plat__azure_msis }}"
+
 
-- name: Wait for MSIs to be delisted
+- name: Wait for MSIs to be delisted - Non RAZ
   when:
     - plat__azure_metagroup_uri is defined
     - plat__azure_metagroup_uri | length > 0
@@ -65,6 +62,26 @@
   vars:
     discovered_msi_list: "{{ __azure_identity_list.response | map(attribute='name') | list }}"
 
+
+- name: Wait for MSIs to be delisted - RAZ
+  when:
+    - plat__azure_metagroup_uri is defined
+    - plat__azure_metagroup_uri | length > 0
+    - plat__enable_raz
+  azure.azcollection.azure_rm_resource_info:
+    resource_group: "{{ plat__azure_metagroup_name }}"
+    provider: ManagedIdentity
+    resource_type: userAssignedIdentities
+    api_version: '2018-11-30'
+  register: __azure_identity_list
+  delay: 5
+  retries: 10
+  until:
+    - plat__azure_raz_identity_name not in discovered_msi_list
+  vars:
+    discovered_msi_list: "{{ __azure_identity_list.response | map(attribute='name') | list }}"
+
+
 - name: Remove CDP Cross Account Credential for Azure
   when: plat__teardown_deletes_credential
   cloudera.cloud.env_cred:

roles/platform/vars/main.yml

@@ -86,6 +86,7 @@ plat__azure_roles:
   storcnt: 'Storage Blob Data Contributor'
   contrib: 'Contributor'
   owner: 'Owner'
+  stordel: 'Storage Blob Delegator'
 
 
 plat__gcp_required_services:
@@ -95,4 +96,4 @@ plat__gcp_required_services:
   - iamcredentials.googleapis.com
   - storage.googleapis.com
   - servicenetworking.googleapis.com
-  - sqladmin.googleapis.com
+  - sqladmin.googleapis.com