diff --git a/cluster.yml b/cluster.yml index 8439680..4c13029 100644 --- a/cluster.yml +++ b/cluster.yml @@ -69,7 +69,7 @@ - full_cluster - name: Verify definition [verify_definition] - hosts: localhost + hosts: cloudera_manager gather_facts: no roles: - cloudera.cluster.verify.definition @@ -92,7 +92,7 @@ # Moved from verify_parcels to reduce duplication - name: Verify definition [verify_parcels_and_roles] - hosts: localhost + hosts: cloudera_manager gather_facts: no roles: - cloudera.cluster.verify.parcels_and_roles @@ -106,7 +106,7 @@ # STARTBLOCK # Prepare Nodes - name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster + hosts: cloudera_manager, cluster, ca_server become: yes roles: - cloudera.cluster.prereqs.os @@ -441,6 +441,7 @@ when: cloudera_manager_agent_wait_for_heartbeat | default(True) tags: - heartbeat + - default_cluster - full_cluster - name: Deploy Cloudera Management Service @@ -466,7 +467,7 @@ - full_cluster - name: Deploy clusters - hosts: localhost + hosts: cloudera_manager gather_facts: no roles: - cloudera.cluster.deployment.cluster diff --git a/roles/cloudera_deploy/defaults/main.yml b/roles/cloudera_deploy/defaults/main.yml index 2331e69..35e16e0 100644 --- a/roles/cloudera_deploy/defaults/main.yml +++ b/roles/cloudera_deploy/defaults/main.yml @@ -43,7 +43,7 @@ use_default_cluster_definition: no # Default parcel cache default_enable_download_mirror: no -default_parcel_distro: el7.parcel +default_parcel_distro: el7 # el8, bionic, focal default_download_link_expiry: 3600 # Default Deployment Controls diff --git a/roles/cloudera_deploy/tasks/init.yml b/roles/cloudera_deploy/tasks/init.yml index 39565d1..fec411c 100644 --- a/roles/cloudera_deploy/tasks/init.yml +++ b/roles/cloudera_deploy/tasks/init.yml @@ -277,6 +277,11 @@ - "Permissions are {{ __private_key_file_stat.stat.mode }}" - "Permissions should be 0400 or 0600" +# Parcel Distro +- name: Determine preferred Parcel Distribution + ansible.builtin.set_fact: + init__parcel_distro: "{{ parcel_distro | default(default_parcel_distro) }}" + # Read in Dynamic Inventory - name: Seek Inventory Template in Definition Path register: __di_template_stat @@ -329,6 +334,7 @@ - dynamic_inventory: vm: count: "{{ __dynamic_inventory_host_list | count }}" + os: "{{ init__parcel_distro }}" always: - name: Remove Dynamic Inventory Template from current inventory include_tasks: refresh_inventory.yml diff --git a/roles/cloudera_deploy/tasks/inject_download_mirror.yml b/roles/cloudera_deploy/tasks/inject_download_mirror.yml index 0851f70..f087f5b 100644 --- a/roles/cloudera_deploy/tasks/inject_download_mirror.yml +++ b/roles/cloudera_deploy/tasks/inject_download_mirror.yml @@ -86,12 +86,26 @@ ansible.builtin.set_fact: init__cluster_repo_entries: "{{ init__cluster_repo_entries | default([]) + [__cluster_repo_item | urlsplit('path') ] }}" - - name: Create list of Download Mirror URLs filtered to required repositories for run + - name: Create list of Download Mirror URLs filtered to required repositories and distros loop: "{{ init__cluster_repo_entries }}" loop_control: loop_var: __cluster_repo_path_item ansible.builtin.set_fact: - init__urls_to_sign: "{{ init__urls_to_sign | default([]) + __download_mirror_ini_entry | select('search', __cluster_repo_path_item) | list }}" + init__urls_to_sign: "{{ init__urls_to_sign + | default([]) + __download_mirror_ini_entry + | select('search', __cluster_repo_path_item) + | select('search', init__parcel_distro) + | list }}" + + - name: Ensure manifest is included in Download Mirror URLs if present + loop: "{{ init__cluster_repo_entries }}" + loop_control: + loop_var: __cluster_repo_path_item + ansible.builtin.set_fact: + init__urls_to_sign: "{{ init__urls_to_sign + | default([]) + __download_mirror_ini_entry + | select('search', 'manifest.json') + | list }}" - name: Get AWS Specific download URIs when: diff --git a/roles/cloudera_deploy/tasks/populate_download_mirror.yml b/roles/cloudera_deploy/tasks/populate_download_mirror.yml index d75300a..c2681a5 100644 --- a/roles/cloudera_deploy/tasks/populate_download_mirror.yml +++ b/roles/cloudera_deploy/tasks/populate_download_mirror.yml @@ -28,7 +28,7 @@ loop: "{{ globals.download_mirror_targets }}" loop_control: loop_var: __mirror_fetch_item - async: 3600 + async: 7200 poll: 0 ansible.builtin.get_url: url: "{{ __mirror_fetch_item }}" @@ -42,8 +42,8 @@ loop_var: __download_async_item register: __async_download_results until: __async_download_results.finished is defined and __async_download_results.finished - delay: 15 - retries: 300 + delay: 30 + retries: 240 async_status: jid: "{{ __download_async_item.ansible_job_id }}" failed_when: @@ -66,18 +66,58 @@ when: globals.infra_type == 'aws' block: # Prepare to sync cache dir to S3 - - name: Prepare host for Python actions + - name: Setup System Rhel8 + when: + - ansible_os_family == 'RedHat' + - ansible_distribution_major_version | int >= 8 become: yes ansible.builtin.package: + lock_timeout: 180 name: "{{ __package_item }}" + update_cache: yes state: present - lock_timeout: 180 + loop_control: + loop_var: __package_item + loop: + - epel-release + - python3 + + - name: Setup system Rhel7 + when: + - ansible_os_family == 'RedHat' + - ansible_distribution_major_version | int < 8 + become: yes + ansible.builtin.package: + name: "{{ __package_item }}" + state: present + lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(180, omit) }}" loop_control: loop_var: __package_item loop: - epel-release - python-pip + - name: Setup system Debian + when: ansible_os_family == "Debian" + block: + - name: enable Debian Repos + become: yes + apt_repository: + repo: "{{ __repo_item }}" + loop_control: + loop_var: __repo_item + loop: + - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }} universe" + - "deb http://archive.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-updates universe" + - "deb http://security.ubuntu.com/ubuntu/ {{ globals.dynamic_inventory.vm.os }}-security universe" + + - name: Install Pip on Debian + become: yes + ansible.builtin.apt: + update_cache: yes + name: python3-pip + state: present + - name: Prepare host for s3 actions become: yes ansible.builtin.pip: @@ -86,7 +126,7 @@ loop_var: __pip_item loop: - futures - - boto3 >= 1.4.4 + - "{{ (ansible_python_version[0] == '2') | ternary('boto3 >= 1.4.4,<1.18', 'boto3 >= 1.20.0') }}" - name: Sync downloaded Files paths to S3 cache bucket become: yes diff --git a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml b/roles/cloudera_deploy/tasks/prepare_download_mirror.yml index 80fed64..10ef568 100644 --- a/roles/cloudera_deploy/tasks/prepare_download_mirror.yml +++ b/roles/cloudera_deploy/tasks/prepare_download_mirror.yml @@ -39,14 +39,16 @@ public: yes vars: repositories: "{{ __init_parcel_repos }}" + cluster_os_distribution: "{{ init__parcel_distro }}" - name: Extract Parcel URLs from Manifests ansible.builtin.set_fact: __parcel_urls: "{{ manifests.results | cloudera.cluster.extract_parcel_urls }}" + __parcel_distro_search_term: "{{ init__parcel_distro }}.parcel" - name: Filter Parcels by distro ansible.builtin.set_fact: - __filtered_parcel_urls: "{{ __parcel_urls | select('search', parcel_distro | default(default_parcel_distro)) | list }}" + __filtered_parcel_urls: "{{ __parcel_urls | select('search', __parcel_distro_search_term ) | list }}" - name: Prepare target Download Mirror listing with parcels and attendant files when: __filtered_parcel_urls | length > 0