Add EL8 support to Cloudera Collections

cluster.yml

@@ -69,7 +69,7 @@
     - full_cluster
 
 - name: Verify definition [verify_definition]
-  hosts: localhost
+  hosts: cloudera_manager
   gather_facts: no
   roles:
     - cloudera.cluster.verify.definition
@@ -92,7 +92,7 @@
 
 # Moved from verify_parcels to reduce duplication
 - name: Verify definition [verify_parcels_and_roles]
-  hosts: localhost
+  hosts: cloudera_manager
   gather_facts: no
   roles:
     - cloudera.cluster.verify.parcels_and_roles
@@ -106,7 +106,7 @@
 # STARTBLOCK # Prepare Nodes
 
 - name: Apply OS pre-requisite configurations
-  hosts: cloudera_manager, cluster
+  hosts: cloudera_manager, cluster, ca_server
   become: yes
   roles:
     - cloudera.cluster.prereqs.os
@@ -441,6 +441,7 @@
       when: cloudera_manager_agent_wait_for_heartbeat | default(True)
   tags:
     - heartbeat
+    - default_cluster
     - full_cluster
 
 - name: Deploy Cloudera Management Service
@@ -466,7 +467,7 @@
     - full_cluster
 
 - name: Deploy clusters
-  hosts: localhost
+  hosts: cloudera_manager
   gather_facts: no
   roles:
     - cloudera.cluster.deployment.cluster

roles/cloudera_deploy/defaults/main.yml

@@ -43,7 +43,7 @@ use_default_cluster_definition: no
 
 # Default parcel cache
 default_enable_download_mirror: no
-default_parcel_distro: el7.parcel
+default_parcel_distro: el7  # el8, bionic, focal
 default_download_link_expiry: 3600
 
 # Default Deployment Controls

roles/cloudera_deploy/tasks/init.yml

@@ -277,6 +277,11 @@
           - "Permissions are {{ __private_key_file_stat.stat.mode }}"
           - "Permissions should be 0400 or 0600"
 
+# Parcel Distro
+- name: Determine preferred Parcel Distribution
+  ansible.builtin.set_fact:
+    init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}"
+
 # Read in Dynamic Inventory
 - name: Seek Inventory Template in Definition Path
   register: __di_template_stat
@@ -329,6 +334,7 @@
         - dynamic_inventory:
             vm:
               count: "{{ __dynamic_inventory_host_list | count }}"
+              os: "{{ init__parcel_distro }}"
   always:
     - name: Remove Dynamic Inventory Template from current inventory
       include_tasks: refresh_inventory.yml

roles/cloudera_deploy/tasks/inject_download_mirror.yml

@@ -86,12 +86,26 @@
       ansible.builtin.set_fact:
         init__cluster_repo_entries: "{{ init__cluster_repo_entries | default([]) + [__cluster_repo_item | urlsplit('path') ] }}"
 
-    - name: Create list of Download Mirror URLs filtered to required repositories for run
+    - name: Create list of Download Mirror URLs filtered to required repositories and distros
       loop: "{{ init__cluster_repo_entries }}"
       loop_control:
         loop_var: __cluster_repo_path_item
       ansible.builtin.set_fact:
-        init__urls_to_sign: "{{ init__urls_to_sign | default([]) + __download_mirror_ini_entry | select('search', __cluster_repo_path_item) | list }}"
+        init__urls_to_sign: "{{ init__urls_to_sign
+          | default([]) + __download_mirror_ini_entry
+          | select('search', __cluster_repo_path_item)
+          | select('search', init__parcel_distro)
+          | list }}"
+
+    - name: Ensure manifest is included in Download Mirror URLs if present
+      loop: "{{ init__cluster_repo_entries }}"
+      loop_control:
+        loop_var: __cluster_repo_path_item
+      ansible.builtin.set_fact:
+        init__urls_to_sign: "{{ init__urls_to_sign
+            | default([]) + __download_mirror_ini_entry
+            | select('search', 'manifest.json')
+            | list }}"
 
 - name: Get AWS Specific download URIs
   when:

roles/cloudera_deploy/tasks/populate_download_mirror.yml

@@ -28,7 +28,7 @@
   loop: "{{ globals.download_mirror_targets }}"
   loop_control:
     loop_var: __mirror_fetch_item
-  async: 3600
+  async: 7200
   poll: 0
   ansible.builtin.get_url:
     url: "{{ __mirror_fetch_item }}"
@@ -42,8 +42,8 @@
     loop_var: __download_async_item
   register: __async_download_results
   until: __async_download_results.finished is defined and __async_download_results.finished
-  delay: 15
-  retries: 300
+  delay: 30
+  retries: 240
   async_status:
     jid: "{{ __download_async_item.ansible_job_id }}"
   failed_when:
@@ -66,18 +66,58 @@
   when: globals.infra_type == 'aws'
   block:
     # Prepare to sync cache dir to S3
-    - name: Prepare host for Python actions
+    - name: Setup System Rhel8
+      when:
+        - ansible_os_family == 'RedHat'
+        - ansible_distribution_major_version | int >= 8
       become: yes
       ansible.builtin.package:
+        lock_timeout: 180
         name: "{{ __package_item }}"
+        update_cache: yes
         state: present
-        lock_timeout: 180
+      loop_control:
+        loop_var: __package_item
+      loop:
+        - epel-release
+        - python3
+
+    - name: Setup system Rhel7
+      when:
+        - ansible_os_family == 'RedHat'
+        - ansible_distribution_major_version | int < 8
+      become: yes
+      ansible.builtin.package:
+        name: "{{ __package_item }}"
+        state: present
+        lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(180, omit) }}"
       loop_control:
         loop_var: __package_item
       loop:
         - epel-release
         - python-pip
 
+    - name: Setup system Debian
+      when: ansible_os_family == "Debian"
+      block:
+        - name: enable Debian Repos
+          become: yes
+          apt_repository:
+            repo: "{{ __repo_item }}"
+          loop_control:
+            loop_var: __repo_item
+          loop:
+            - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }} universe"
+            - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-updates universe"
+            - "deb http://security.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-security universe"
+
+        - name: Install Pip on Debian
+          become: yes
+          ansible.builtin.apt:
+            update_cache: yes
+            name: python3-pip
+            state: present
+
     - name: Prepare host for s3 actions
       become: yes
       ansible.builtin.pip:
@@ -86,7 +126,7 @@
         loop_var: __pip_item
       loop:
         - futures
-        - boto3 >= 1.4.4
+        - "{{ (ansible_python_version[0] == '2') | ternary('boto3 >= 1.4.4,<1.18', 'boto3 >= 1.20.0') }}"
 
     - name: Sync downloaded Files paths to S3 cache bucket
       become: yes

roles/cloudera_deploy/tasks/prepare_download_mirror.yml

@@ -39,14 +39,16 @@
     public: yes
   vars:
     repositories: "{{ __init_parcel_repos }}"
+    cluster_os_distribution: "{{ init__parcel_distro }}"
 
 - name: Extract Parcel URLs from Manifests
   ansible.builtin.set_fact:
     __parcel_urls: "{{ manifests.results | cloudera.cluster.extract_parcel_urls }}"
+    __parcel_distro_search_term: "{{ init__parcel_distro }}.parcel"
 
 - name: Filter Parcels by distro
   ansible.builtin.set_fact:
-    __filtered_parcel_urls: "{{ __parcel_urls | select('search', parcel_distro | default(default_parcel_distro)) | list }}"
+    __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}"
 
 - name: Prepare target Download Mirror listing with parcels and attendant files
   when: __filtered_parcel_urls | length > 0