Skip to content

Commit 762166e

Browse files
tmgstevenstmgstevens
authored
Pvc experiences (#44)
Include support for PvC experiences. Things still outstanding: Teardown TLS (getting truststores and jks base64 encoded and into the yaml file) Prometheus ht credentials Testing on a real cluster Proposal is to do these under a separate PR, instead get this PR into devel for people to start using
1 parent cd8aeba commit 762166e

File tree

16 files changed

+716
-0
lines changed

16 files changed

+716
-0
lines changed

‎roles/config/cluster/common/defaults/main.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@ default_cluster_kts:
3737
KEYTRUSTEE_PASSIVE_SERVER: {}
3838
DB_PASSIVE: {}
3939

40+
default_cluster_ecs:
41+
name: ECS
42+
services: [DOCKER, ECS]
43+
44+
ecs_databases: [ALERTS, CLASSIC_CLUSTERS, CLUSTER_ACCESS_MANAGER, CLUSTER_PROXY, DEX, DWX, ENV, LIFTIE, MLX, RESOURCEPOOL_MANAGER, UMS]
45+
4046
default_cluster_compute:
4147
base_cluster:
4248
data_context: SDX

‎roles/config/cluster/ecs/meta/main.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2021 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
dependencies:
18+
- role: cloudera.cluster.cloudera_manager.common
19+
- role: cloudera.cluster.deployment.definition
20+
- role: cloudera.cluster.config.cluster.common
21+
- role: cloudera.cluster.infrastructure.ca_common

‎roles/config/cluster/ecs/tasks/main.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2021 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
# This variable is used by other roles
18+
# please take care when changing it
19+
- set_fact:
20+
databases: "{{ database_defaults | combine(cluster.databases | default({}), recursive=True) }}"
21+
22+
- name: Retrieve repository metadata
23+
include_role:
24+
name: cloudera.cluster.deployment.repometa
25+
vars:
26+
repositories: "{{ cluster.repositories | default({}) }}"
27+
28+
- name: Reset custom configuration dictionary
29+
set_fact:
30+
custom_configs: {}
31+
32+
- name: Generate custom configurations
33+
set_fact:
34+
custom_configs: "{{ custom_configs | combine(lookup('template', custom_config_template.template) | from_yaml, recursive=True) }}"
35+
loop: "{{ custom_config_templates }}"
36+
loop_control:
37+
loop_var: custom_config_template
38+
when: custom_config_template.condition | default(True)
39+
40+
# This variable is used by other roles
41+
# please take care when changing it
42+
- name: Merge custom configurations
43+
set_fact:
44+
merged_configs: "{{ custom_configs | combine(cluster.configs | default({}), recursive=True) }}"

‎roles/config/cluster/ecs/templates/configs/ecs.j2‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
DOCKER:
3+
SERVICEWIDE:
4+
docker_images_destination_registry_user: registry-user
5+
ECS:
6+
SERVICEWIDE:
7+
k8s_webui_secret_admin_token: ecs-k8s_webui_secret_admin_token
8+
cp_prometheus_ingress_user: cloudera-manager
9+
infra_prometheus_ingress_user: cloudera-manager
10+
longhorn_replication: 1
11+
lsoDataPath: /ecs/storage
12+
app_domain: "{{ cluster.application_domain }}"

‎roles/config/cluster/ecs/vars/main.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2021 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
custom_config_templates:
18+
# Custom configurations for ECS
19+
- template: configs/ecs.j2

‎roles/deployment/cluster/defaults/main.yml‎

Whitespace-only changes.

‎roles/deployment/cluster/meta/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
---
1616
dependencies:
1717
- role: cloudera.cluster.cloudera_manager.api_hosts
18+
- role: cloudera.cluster.deployment.credential
1819
- role: cloudera.cluster.infrastructure.ca_common
1920
- role: cloudera.cluster.prereqs.local_accounts_common
2021

‎roles/deployment/cluster/tasks/create_ecs.yml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# Copyright 2021 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
17+
- name: Generate complete ecs cluster configs
18+
include_role:
19+
name: cloudera.cluster.config.cluster.ecs
20+
21+
- name: Create databases and users
22+
include_role:
23+
name: cloudera.cluster.deployment.databases
24+
vars:
25+
services: "{{ cluster.ecs_databases | default(ecs_databases) }}"
26+
27+
# Add PVC parcel repo - assume that this has been set via REMOTE_PARCEL_REPO_URLS in the cm config. We can add this in later if needed
28+
29+
- name: Create ECS cluster
30+
cloudera.cluster.cm_api:
31+
endpoint: /clusters
32+
method: POST
33+
body: "{{ lookup('template', 'cluster_template/ecs/clusters.j2', convert_data=False) }}"
34+
register: cluster_creation_result
35+
failed_when: '"Status code was 400" in cluster_creation_result.msg'
36+
when: not cluster_template_dry_run
37+
38+
- name: Add hosts to ECS cluster
39+
cloudera.cluster.cm_api:
40+
endpoint: /clusters/{{ cluster.name | urlencode() }}/hosts
41+
method: POST
42+
body: "{{ lookup('template', 'cluster_template/ecs/hosts.j2', convert_data=False) }}"
43+
register: cluster_hosts_result
44+
failed_when: '"Status code was 400" in cluster_hosts_result.msg'
45+
when: not cluster_template_dry_run
46+
47+
- name: Add services to ECS cluster
48+
cloudera.cluster.cm_api:
49+
endpoint: /clusters/{{ cluster.name | urlencode() }}/services
50+
method: POST
51+
body: "{{ lookup('template', 'cluster_template/ecs/services.j2', convert_data=False) }}"
52+
register: cluster_services_result
53+
failed_when: '"Status code was 400" in cluster_services_result.msg'
54+
when: not cluster_template_dry_run
55+
56+
- name: Get CM config
57+
cloudera.cluster.cm_api:
58+
endpoint: "cm/config"
59+
method: GET
60+
register: cm_config
61+
62+
- set_fact:
63+
parcel_repos: "{{ (cm_config.json | json_query('items[?name==`REMOTE_PARCEL_REPO_URLS`].value') | default(['']))[0].split(',') | default([]) | union(cluster.repositories) }}"
64+
65+
- name: Update parcelrepos
66+
cloudera.cluster.cm_api:
67+
endpoint: "cm/config"
68+
body:
69+
items:
70+
- name: REMOTE_PARCEL_REPO_URLS
71+
value: "{{ parcel_repos | join(', ') }}"
72+
method: PUT
73+
74+
- name: Refresh parcel repos command
75+
cloudera.cluster.cm_api:
76+
endpoint: "/cm/commands/refreshParcelRepos"
77+
method: POST
78+
79+
- set_fact:
80+
new_parcel_version: "{{ products | json_query('[?product==`ECS`].version') | first }}"
81+
82+
- name: Start Parcel Download
83+
cloudera.cluster.cm_api:
84+
endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDownload"
85+
method: POST
86+
87+
- name: Wait for parcels to download
88+
cloudera.cluster.cm_api:
89+
endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
90+
register: parcels_response
91+
until: parcels_response.json.stage in ("DOWNLOADED", "DISTRIBUTED", "ACTIVATED")
92+
retries: "{{ parcel_poll_max_retries | default(30) }}"
93+
delay: "{{ parcel_poll_duration | default(60) }}"
94+
95+
- name: Start Parcel Distribution
96+
cloudera.cluster.cm_api:
97+
endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDistribution"
98+
method: POST
99+
100+
- name: Wait for parcels to become distributed to all hosts
101+
cloudera.cluster.cm_api:
102+
endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
103+
register: parcels_response
104+
until: parcels_response.json.stage in ("DISTRIBUTED", "ACTIVATED")
105+
retries: "{{ parcel_poll_max_retries | default(30) }}"
106+
delay: "{{ parcel_poll_duration | default(60) }}"
107+
108+
- name: Activate Parcels
109+
cloudera.cluster.cm_api:
110+
endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/activate"
111+
method: POST
112+
113+
- name: Wait for parcels to become distributed to all hosts
114+
cloudera.cluster.cm_api:
115+
endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
116+
register: parcels_response
117+
until: parcels_response.json.stage in ("ACTIVATED")
118+
retries: "{{ parcel_poll_max_retries | default(30) }}"
119+
delay: "{{ parcel_poll_duration | default(60) }}"
120+
121+
- name: Generate custom values
122+
set_fact:
123+
custom_values: "{{ lookup('template', 'cluster_template/ecs/controlPlaneValues.j2') | from_yaml | combine(cluster.controlplane_config, recursive=True) }}"
124+
125+
- name: Show custom values
126+
debug:
127+
var: custom_values
128+
129+
- name: Install ECS cluster
130+
cloudera.cluster.cm_api:
131+
endpoint: /controlPlanes/commands/installEmbeddedControlPlane
132+
method: POST
133+
body:
134+
remoteRepoUrl: "{{ cluster.pvc_repository }}"
135+
experienceClusterName: "{{ cluster.name }}"
136+
datalakeClusterName: "{{ cluster.datalake }}"
137+
valuesYaml: "{{ custom_values | to_yaml }}"
138+
register: controlplane_response
139+

‎roles/deployment/cluster/tasks/main.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,18 @@
114114
- cluster.type | default(default_cluster_type) == 'compute'
115115
- cluster.name not in existing_clusters
116116

117+
- name: Create ECS clusters
118+
include_tasks: create_ecs.yml
119+
loop: "{{ definition.clusters }}"
120+
loop_control:
121+
loop_var: _cluster
122+
vars:
123+
cluster: "{{ default_cluster_ecs | combine(_cluster) }}"
124+
when:
125+
- cluster.type | default(default_cluster_type) == 'ecs'
126+
- cluster.name not in existing_clusters
127+
128+
117129
- name: Restart Cloudera Management Service
118130
cloudera.cluster.cm_api:
119131
endpoint: /cm/service/commands/restart

‎roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"items":
3+
[
4+
{
5+
"name" : "{{ cluster.name }}",
6+
"displayName": "{{ cluster.name }}",
7+
"version" : "EXPERIENCE1",
8+
"fullVersion": "{{ products | cloudera.cluster.get_product_version('ECS') }}",
9+
"clusterType": "EXPERIENCE_CLUSTER"
10+
}
11+
]
12+
}

0 commit comments

Comments
 (0)