[Tests] Update cluster-api provider to use machineTemplate.status.nodeInfo for architecture-aware autoscale from zero

kubernetes-sigs/cluster-api#11962 introduced the nodeInfo field for MachineTemplates. Providers can reconcile this field in the status subresource to inform the autoscaler about the architecture and operating system that the MachineTemplate's nodes will run.

Previously, we have been implementing this behavior in the cluster autoscaler by leveraging the labels capacity annotation and, as a fallback, default values set in environment variables at cluster-autoscaler deployment time.

With this commit, the cluster autoscaler computes the future architecture of a node with the following priority order:

- Labels set in existing nodes for not-autoscale-from-zero cases
- Labels set in the labels capacity annotation of machine template, machine set, and machine deployment.
- Values in the status.nodeSystemInfo of MachineTemplates
- Generic/default labels set in the environment of the cluster autoscaler

Author: aleskandro

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_test_framework.go

@@ -60,6 +60,7 @@ type testConfigBuilder struct {
 	nodeCount    int
 	annotations  map[string]string
 	capacity     map[string]string
+	nodeInfo     map[string]string
 }
 
 // NewTestConfigBuilder returns a builder for dynamically constructing mock ClusterAPI resources for testing.
@@ -91,6 +92,7 @@ func (b *testConfigBuilder) Build() *TestConfig {
 			isMachineDeployment,
 			b.annotations,
 			b.capacity,
+			b.nodeInfo,
 		)[0],
 	)[0]
 }
@@ -111,6 +113,7 @@ func (b *testConfigBuilder) BuildMultiple(configCount int) []*TestConfig {
 			isMachineDeployment,
 			b.annotations,
 			b.capacity,
+			b.nodeInfo,
 		)...,
 	)
 }
@@ -171,6 +174,18 @@ func (b *testConfigBuilder) WithCapacity(c map[string]string) *testConfigBuilder
 	return b
 }
 
+func (b *testConfigBuilder) WithNodeInfo(n map[string]string) *testConfigBuilder {
+	if n == nil {
+		b.nodeInfo = nil
+	} else {
+		if b.nodeInfo == nil {
+			b.nodeInfo = map[string]string{}
+		}
+		maps.Insert(b.nodeInfo, maps.All(n))
+	}
+	return b
+}
+
 // TestConfig contains clusterspecific information about a single test configuration.
 type TestConfig struct {
 	spec              *TestSpec
@@ -290,8 +305,8 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
 			UID:  config.machineSet.GetUID(),
 		}
 
-		if spec.capacity != nil {
-			klog.V(4).Infof("adding capacity to machine template")
+		if spec.capacity != nil || spec.nodeInfo != nil {
+			klog.V(4).Infof("creating machine template")
 			config.machineTemplate = &unstructured.Unstructured{
 				Object: map[string]interface{}{
 					"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
@@ -303,13 +318,25 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
 					},
 				},
 			}
+		}
+		if spec.capacity != nil {
+			klog.V(4).Infof("adding capacity to machine template")
 			if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.capacity, "status", "capacity"); err != nil {
 				panic(err)
 			}
 		} else {
 			klog.V(4).Infof("not adding capacity")
 		}
 
+		if spec.nodeInfo != nil {
+			klog.V(4).Infof("adding node info")
+			if err := unstructured.SetNestedStringMap(config.machineTemplate.Object, spec.nodeInfo, "status", "nodeInfo"); err != nil {
+				panic(err)
+			}
+		} else {
+			klog.V(4).Infof("not adding node info")
+		}
+
 		for j := 0; j < spec.nodeCount; j++ {
 			config.nodes[j], config.machines[j] = makeLinkedNodeAndMachine(j, spec.namespace, spec.clusterName, machineOwner, machineSetLabels)
 		}
@@ -324,6 +351,7 @@ func createTestConfigs(specs ...TestSpec) []*TestConfig {
 type TestSpec struct {
 	annotations             map[string]string
 	capacity                map[string]string
+	nodeInfo                map[string]string
 	machineDeploymentName   string
 	machineSetName          string
 	machinePoolName         string
@@ -333,17 +361,17 @@ type TestSpec struct {
 	rootIsMachineDeployment bool
 }
 
-func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) []TestSpec {
+func createTestSpecs(namespace, clusterName, namePrefix string, scalableResourceCount, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) []TestSpec {
 	var specs []TestSpec
 
 	for i := 0; i < scalableResourceCount; i++ {
-		specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity))
+		specs = append(specs, createTestSpec(namespace, clusterName, fmt.Sprintf("%s-%d", namePrefix, i), nodeCount, isMachineDeployment, annotations, capacity, nodeInfo))
 	}
 
 	return specs
 }
 
-func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string) TestSpec {
+func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachineDeployment bool, annotations map[string]string, capacity map[string]string, nodeInfo map[string]string) TestSpec {
 	return TestSpec{
 		annotations:             annotations,
 		capacity:                capacity,
@@ -353,6 +381,7 @@ func createTestSpec(namespace, clusterName, name string, nodeCount int, isMachin
 		namespace:               namespace,
 		nodeCount:               nodeCount,
 		rootIsMachineDeployment: isMachineDeployment,
+		nodeInfo:                nodeInfo,
 	}
 }
 

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_unstructured_test.go

@@ -33,9 +33,14 @@ import (
 )
 
 const (
-	cpuStatusKey       = "cpu"
-	memoryStatusKey    = "memory"
-	nvidiaGpuStatusKey = "nvidia.com/gpu"
+	cpuStatusKey             = "cpu"
+	memoryStatusKey          = "memory"
+	nvidiaGpuStatusKey       = "nvidia.com/gpu"
+	architectureStatusKey    = "architecture"
+	operatingSystemStatusKey = "operatingSystem"
+
+	arm64 = "arm64"
+	linux = "linux"
 )
 
 func TestSetSize(t *testing.T) {
@@ -595,3 +600,110 @@ func TestCanScaleFromZero(t *testing.T) {
 		})
 	}
 }
+
+func TestInstanceSystemInfo(t *testing.T) {
+	// use a constant capacity as that's necessary for the business logic to consider the resource scalable
+	capacity := map[string]string{
+		cpuStatusKey:    "1",
+		memoryStatusKey: "4G",
+	}
+	testConfigs := []struct {
+		name         string
+		nodeInfo     map[string]string
+		expectedArch string
+		expectedOS   string
+	}{
+		{
+			"with no architecture or operating system in machine template's status' nodeInfo, the system info is empty",
+			map[string]string{},
+			"",
+			"",
+		},
+		{
+			"with architecture in machine template's status' nodeInfo, the system info is filled in the scalable resource",
+			map[string]string{
+				architectureStatusKey: arm64,
+			},
+			arm64,
+			"",
+		},
+		{
+			"with operating system in machine template's status' nodeInfo, the system info is filled in the scalable resource",
+			map[string]string{
+				operatingSystemStatusKey: linux,
+			},
+			"",
+			linux,
+		},
+		{
+			"with architecture and operating system in machine template's status' nodeInfo, the system info is filled in the scalable resource",
+			map[string]string{
+				architectureStatusKey:    arm64,
+				operatingSystemStatusKey: linux,
+			},
+			arm64,
+			linux,
+		},
+	}
+
+	for _, tc := range testConfigs {
+		testname := fmt.Sprintf("MachineSet %s", tc.name)
+		t.Run(testname, func(t *testing.T) {
+			mdTestConfig := NewTestConfigBuilder().
+				ForMachineSet().
+				WithNodeCount(1).
+				WithCapacity(capacity).
+				WithNodeInfo(tc.nodeInfo).
+				Build()
+			controller := NewTestMachineController(t)
+			defer controller.Stop()
+			controller.AddTestConfigs(mdTestConfig)
+
+			testResource := mdTestConfig.machineSet
+
+			sr, err := newUnstructuredScalableResource(controller.machineController, testResource)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			sysInfo := sr.InstanceSystemInfo()
+			if sysInfo.Architecture != tc.expectedArch {
+				t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture)
+			}
+			if sysInfo.OperatingSystem != tc.expectedOS {
+				t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem)
+			}
+		})
+	}
+
+	for _, tc := range testConfigs {
+		testname := fmt.Sprintf("MachineDeployment %s", tc.name)
+		t.Run(testname, func(t *testing.T) {
+			mdTestConfig := NewTestConfigBuilder().
+				ForMachineDeployment().
+				WithNodeCount(1).
+				WithCapacity(capacity).
+				WithNodeInfo(tc.nodeInfo).
+				Build()
+			controller := NewTestMachineController(t)
+			defer controller.Stop()
+			controller.AddTestConfigs(mdTestConfig)
+
+			testResource := mdTestConfig.machineDeployment
+
+			sr, err := newUnstructuredScalableResource(controller.machineController, testResource)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			sysInfo := sr.InstanceSystemInfo()
+			if sysInfo.Architecture != tc.expectedArch {
+				t.Errorf("expected architecture %s, got %s", tc.nodeInfo[architectureStatusKey], sysInfo.Architecture)
+			}
+
+			if sysInfo.OperatingSystem != tc.expectedOS {
+				t.Errorf("expected operating system %s, got %s", tc.nodeInfo[operatingSystemStatusKey], sysInfo.OperatingSystem)
+			}
+		})
+	}
+}