Skip to content

Commit b27f5d6

Browse files
aleskandroaleskandro
committed
[Tests] Update cluster-api provider to use machineTemplate.status.nodeInfo for architecture-aware autoscale from zero
kubernetes-sigs/cluster-api#11962 introduced the nodeInfo field for MachineTemplates. Providers can reconcile this field in the status subresource to inform the autoscaler about the architecture and operating system that the MachineTemplate's nodes will run. Previously, we have been implementing this behavior in the cluster autoscaler by leveraging the labels capacity annotation and, as a fallback, default values set in environment variables at cluster-autoscaler deployment time. With this commit, the cluster autoscaler computes the future architecture of a node with the following priority order: - Labels set in existing nodes for not-autoscale-from-zero cases - Labels set in the labels capacity annotation of machine template, machine set, and machine deployment. - Values in the status.nodeSystemInfo of MachineTemplates - Generic/default labels set in the environment of the cluster autoscaler # Conflicts: # cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go # cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go # cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider_test.go # cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go
1 parent 1d22638 commit b27f5d6

File tree

4 files changed

+167
-26
lines changed

4 files changed

+167
-26
lines changed

‎cluster-autoscaler/cloudprovider/clusterapi/clusterapi_autodiscovery_test.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -204,17 +204,17 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
204204
shouldMatch bool
205205
}{{
206206
name: "no clustername, namespace, or label selector specified should match any MachineSet",
207-
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil),
207+
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil),
208208
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
209209
shouldMatch: true,
210210
}, {
211211
name: "no clustername, namespace, or label selector specified should match any MachineDeployment",
212-
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil),
212+
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil),
213213
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{labelSelector: labels.NewSelector()},
214214
shouldMatch: true,
215215
}, {
216216
name: "clustername specified does not match MachineSet, namespace matches, no labels specified",
217-
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil),
217+
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, false, nil, nil, nil),
218218
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
219219
clusterName: "foo",
220220
namespace: "default",
@@ -223,7 +223,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
223223
shouldMatch: false,
224224
}, {
225225
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
226-
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil),
226+
testSpec: createTestSpec("default", RandomString(6), RandomString(6), 1, true, nil, nil, nil),
227227
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
228228
clusterName: "foo",
229229
namespace: "default",
@@ -232,7 +232,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
232232
shouldMatch: false,
233233
}, {
234234
name: "namespace specified does not match MachineSet, clusterName matches, no labels specified",
235-
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil),
235+
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, false, nil, nil, nil),
236236
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
237237
clusterName: "foo",
238238
namespace: "default",
@@ -241,7 +241,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
241241
shouldMatch: false,
242242
}, {
243243
name: "clustername specified does not match MachineDeployment, namespace matches, no labels specified",
244-
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil),
244+
testSpec: createTestSpec(RandomString(6), "foo", RandomString(6), 1, true, nil, nil, nil),
245245
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
246246
clusterName: "foo",
247247
namespace: "default",
@@ -250,7 +250,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
250250
shouldMatch: false,
251251
}, {
252252
name: "namespace and clusterName matches MachineSet, no labels specified",
253-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
253+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
254254
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
255255
clusterName: "foo",
256256
namespace: "default",
@@ -259,7 +259,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
259259
shouldMatch: true,
260260
}, {
261261
name: "namespace and clusterName matches MachineDeployment, no labels specified",
262-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
262+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
263263
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
264264
clusterName: "foo",
265265
namespace: "default",
@@ -268,7 +268,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
268268
shouldMatch: true,
269269
}, {
270270
name: "namespace and clusterName matches MachineSet, does not match label selector",
271-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
271+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
272272
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
273273
clusterName: "foo",
274274
namespace: "default",
@@ -277,7 +277,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
277277
shouldMatch: false,
278278
}, {
279279
name: "namespace and clusterName matches MachineDeployment, does not match label selector",
280-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
280+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
281281
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
282282
clusterName: "foo",
283283
namespace: "default",
@@ -286,7 +286,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
286286
shouldMatch: false,
287287
}, {
288288
name: "namespace, clusterName, and label selector matches MachineSet",
289-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
289+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
290290
additionalLabels: map[string]string{"color": "green"},
291291
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
292292
clusterName: "foo",
@@ -296,7 +296,7 @@ func Test_allowedByAutoDiscoverySpec(t *testing.T) {
296296
shouldMatch: true,
297297
}, {
298298
name: "namespace, clusterName, and label selector matches MachineDeployment",
299-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
299+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
300300
additionalLabels: map[string]string{"color": "green"},
301301
autoDiscoveryConfig: &clusterAPIAutoDiscoveryConfig{
302302
clusterName: "foo",

‎cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,23 +1288,23 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
12881288
shouldMatch bool
12891289
}{{
12901290
name: "autodiscovery specs includes permissive spec that should match any MachineSet",
1291-
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil),
1291+
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, false, nil, nil, nil),
12921292
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
12931293
{labelSelector: labels.NewSelector()},
12941294
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
12951295
},
12961296
shouldMatch: true,
12971297
}, {
12981298
name: "autodiscovery specs includes permissive spec that should match any MachineDeployment",
1299-
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil),
1299+
testSpec: createTestSpec(RandomString(6), RandomString(6), RandomString(6), 1, true, nil, nil, nil),
13001300
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
13011301
{labelSelector: labels.NewSelector()},
13021302
{clusterName: "foo", namespace: "bar", labelSelector: labels.Nothing()},
13031303
},
13041304
shouldMatch: true,
13051305
}, {
13061306
name: "autodiscovery specs includes a restrictive spec that should match specific MachineSet",
1307-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
1307+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
13081308
additionalLabels: map[string]string{"color": "green"},
13091309
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
13101310
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
@@ -1313,7 +1313,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
13131313
shouldMatch: true,
13141314
}, {
13151315
name: "autodiscovery specs includes a restrictive spec that should match specific MachineDeployment",
1316-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
1316+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
13171317
additionalLabels: map[string]string{"color": "green"},
13181318
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
13191319
{clusterName: "foo", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "green"})},
@@ -1322,7 +1322,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
13221322
shouldMatch: true,
13231323
}, {
13241324
name: "autodiscovery specs does not include any specs that should match specific MachineSet",
1325-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil),
1325+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, false, nil, nil, nil),
13261326
additionalLabels: map[string]string{"color": "green"},
13271327
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
13281328
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},
@@ -1331,7 +1331,7 @@ func Test_machineController_allowedByAutoDiscoverySpecs(t *testing.T) {
13311331
shouldMatch: false,
13321332
}, {
13331333
name: "autodiscovery specs does not include any specs that should match specific MachineDeployment",
1334-
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil),
1334+
testSpec: createTestSpec("default", "foo", RandomString(6), 1, true, nil, nil, nil),
13351335
additionalLabels: map[string]string{"color": "green"},
13361336
autoDiscoverySpecs: []*clusterAPIAutoDiscoveryConfig{
13371337
{clusterName: "test", namespace: "default", labelSelector: labels.SelectorFromSet(labels.Set{"color": "blue"})},

‎cluster-autoscaler/cloudprovider/clusterapi/clusterapi_test_framework.go‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ type testConfigBuilder struct {
6060
nodeCount int
6161
annotations map[string]string
6262
capacity map[string]string
63+
nodeInfo map[string]string
6364
}
6465

6566
// NewTestConfigBuilder returns a builder for dynamically constructing mock ClusterAPI resources for testing.
@@ -91,6 +92,7 @@ func (b *testConfigBuilder) Build() *TestConfig {
9192
isMachineDeployment,
9293
b.annotations,
9394
b.capacity,
95+
b.nodeInfo,
9496
)[0],
9597
)[0]
9698
}
@@ -111,6 +113,7 @@ func (b *testConfigBuilder) BuildMultiple(configCount int) []*TestConfig {
111113
isMachineDeployment,
112114
b.annotations,
113115
b.capacity,
116+
b.nodeInfo,
114117
)...,
115118
)
116119
}
@@ -171,6 +174,18 @@ func (b *testConfigBuilder) WithCapacity(c map[string]string) *testConfigBuilder
171174
return b
172175
}
173176

177+
func (b *testConfigBuilder) WithNodeInfo(n map[string]string) *testConfigBuilder {
178+
if n == nil {
179+
b.nodeInfo = nil
180+
} else {
181+
if b.nodeInfo == nil {
182+
b.nodeInfo = map[string]string{}
183+
}
184+
maps.Insert(b.nodeInfo, maps.All(n))
185+
}
186+
return b
187+
}
188+
174189
// TestConfig contains clusterspecific information about a single test configuration.
175190
type TestConfig struct {
176191
spec *TestSpec
@@ -290,8 +305,8 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
290305
UID: config.machineSet.GetUID(),
291306
}
292307

293-
if spec.capacity != nil {
294-
klog.V(4).Infof("adding capacity to machine template")
308+
if spec.capacity != nil || spec.nodeInfo != nil {
309+
klog.V(4).Infof("creating machine template")
295310
config.machineTemplate = &unstructured.Unstructured{
296311
Object: map[string]interface{}{
297312
"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
@@ -303,13 +318,25 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
303318
},
304319
},
305320
}
321+
}
322+
if spec.capacity != nil {
323+
klog.V(4).Infof("adding capacity to machine template")
306324
if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.capacity, "status", "capacity"); err != nil {
307325
panic(err)
308326
}
309327
} else {
310328
klog.V(4).Infof("not adding capacity")
311329
}
312330

331+
if spec.nodeInfo != nil {
332+
klog.V(4).Infof("adding node info")
333+
if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.nodeInfo, "status", "nodeInfo"); err != nil {
334+
panic(err)
335+
}
336+
} else {
337+
klog.V(4).Infof("not adding node info")
338+
}
339+
313340
for j := 0; j < spec.nodeCount; j++ {
314341
config.nodes[j], config.machines[j] = makeLinkedNodeAndMachine(j, spec.namespace, spec.clusterName, machineOwner, machineSetLabels)
315342
}
@@ -324,6 +351,7 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
324351
type TestSpec struct {
325352
annotations map[string]string
326353
capacity map[string]string
354+
nodeInfo map[string]string
327355
machineDeploymentName string
328356
machineSetName string
329357
machinePoolName string
@@ -333,17 +361,17 @@ type TestSpec struct {
333361
rootIsMachineDeployment bool
334362
}
335363

336-
func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) []TestSpec {
364+
func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []TestSpec {
337365
var specs []TestSpec
338366

339367
for i := 0; i < scalableResourceCount; i++ {
340-
specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity))
368+
specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity, nodeInfo))
341369
}
342370

343371
return specs
344372
}
345373

346-
func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) TestSpec {
374+
func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) TestSpec {
347375
return TestSpec{
348376
annotations: annotations,
349377
capacity: capacity,
@@ -353,6 +381,7 @@ func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachin
353381
namespace: namespace,
354382
nodeCount: nodeCount,
355383
rootIsMachineDeployment: isMachineDeployment,
384+
nodeInfo: nodeInfo,
356385
}
357386
}
358387

0 commit comments

Comments
 (0)