Skip to content

Commit 2c5bc46

Browse files
kellyyankellyyan
committed
Add support for service weighted target groups
1 parent 32eb0c8 commit 2c5bc46

16 files changed

+1090
-173
lines changed

‎controllers/service/service_controller.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,11 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe
5050
annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
5151
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
5252
serviceUtils := service.NewServiceUtils(annotationParser, shared_constants.ServiceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
53+
enhancedBackendBuilder := service.NewDefaultEnhancedBackendBuilder(k8sClient, annotationParser, logger)
5354
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
5455
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
5556
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
56-
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector, controllerConfig.FeatureGates.Enabled(config.EnableTCPUDPListenerType))
57+
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector, controllerConfig.FeatureGates.Enabled(config.EnableTCPUDPListenerType), enhancedBackendBuilder)
5758
stackMarshaller := deploy.NewDefaultStackMarshaller()
5859
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingManager, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger, metricsCollector, controllerName, controllerConfig.FeatureGates.Enabled(config.EnhancedDefaultBehavior), targetGroupCollector, false)
5960
return &serviceReconciler{

‎docs/guide/service/annotations.md‎

Lines changed: 85 additions & 48 deletions
Large diffs are not rendered by default.

‎pkg/service/config_types.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package service
2+
3+
import (
4+
"github.com/pkg/errors"
5+
"k8s.io/apimachinery/pkg/util/intstr"
6+
)
7+
8+
// TargetGroupTuple Information about how traffic will be distributed between multiple target groups in a forward rule.
9+
// Target group protocol defaults to listener protocol.
10+
type TargetGroupTuple struct {
11+
// The Amazon Resource Name (ARN) of the target group.
12+
TargetGroupARN *string `json:"targetGroupARN"`
13+
14+
// The K8s service Name.
15+
ServiceName *string `json:"serviceName"`
16+
17+
// The K8s service port.
18+
ServicePort *intstr.IntOrString `json:"servicePort"`
19+
20+
// Whether the traffic should be decrypted and forwarded to the target unencrypted. For a TLS listener, its target groups default to TLS protocol. To set the target group to TCP protocol. Set Decrypt to true.
21+
// +optional
22+
Decrypt *bool `json:"decrypt,omitempty"`
23+
24+
// The weight.
25+
// +kubebuilder:validation:Minimum=0
26+
// +kubebuilder:validation:Maximum=999
27+
// +optional
28+
Weight *int32 `json:"weight,omitempty"`
29+
}
30+
31+
func (t *TargetGroupTuple) validate() error {
32+
if (t.TargetGroupARN != nil) == (t.ServiceName != nil) {
33+
return errors.New("precisely one of targetGroupARN and serviceName can be specified")
34+
}
35+
36+
if t.ServiceName != nil && t.ServicePort == nil {
37+
return errors.New("missing servicePort")
38+
}
39+
40+
if t.Weight != nil && (*t.Weight < 0 || *t.Weight > 999) {
41+
return errors.New("target group weight must be between 0 and 999")
42+
}
43+
return nil
44+
}
45+
46+
// TargetGroupStickinessConfig Information about the target group stickiness.
47+
type TargetGroupStickinessConfig struct {
48+
// Whether target group stickiness is enabled.
49+
// +optional
50+
Enabled *bool `json:"enabled,omitempty"`
51+
}
52+
53+
// ForwardActionConfig Information about a forward action.
54+
type ForwardActionConfig struct {
55+
// The weight of the base service.
56+
// +kubebuilder:validation:Minimum=0
57+
// +kubebuilder:validation:Maximum=999
58+
BaseServiceWeight *int32 `json:"baseServiceWeight"`
59+
60+
// One or more target groups.
61+
// +kubebuilder:validation:MaxProperties=4
62+
TargetGroups []TargetGroupTuple `json:"targetGroups"`
63+
64+
// The target group stickiness.
65+
// +optional
66+
TargetGroupStickinessConfig *TargetGroupStickinessConfig `json:"targetGroupStickinessConfig,omitempty"`
67+
}
68+
69+
func (c *ForwardActionConfig) validate() error {
70+
for _, t := range c.TargetGroups {
71+
if err := t.validate(); err != nil {
72+
return errors.Wrap(err, "invalid TargetGroupTuple")
73+
}
74+
}
75+
if len(c.TargetGroups) > 1 {
76+
for _, t := range c.TargetGroups {
77+
if t.Weight == nil {
78+
return errors.New("weight must be set when routing to multiple target groups")
79+
}
80+
}
81+
}
82+
return nil
83+
}
84+
85+
// ActionType The type of action.
86+
type ActionType string
87+
88+
const (
89+
ActionTypeForward ActionType = "forward"
90+
)
91+
92+
type Action struct {
93+
// The type of action.
94+
Type ActionType `json:"type"`
95+
96+
// Information for creating an action that distributes requests among one or more target groups.
97+
ForwardConfig *ForwardActionConfig `json:"forwardConfig,omitempty"`
98+
}
99+
100+
func (a *Action) validate() error {
101+
switch a.Type {
102+
case ActionTypeForward:
103+
if a.ForwardConfig != nil {
104+
if err := a.ForwardConfig.validate(); err != nil {
105+
return errors.Wrap(err, "invalid ForwardConfig")
106+
}
107+
}
108+
default:
109+
return errors.Errorf("unknown action type: %v", a.Type)
110+
}
111+
return nil
112+
}

‎pkg/service/enhanced_backend_builder.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package service
2+
3+
import (
4+
"context"
5+
6+
awssdk "github.com/aws/aws-sdk-go-v2/aws"
7+
"github.com/go-logr/logr"
8+
corev1 "k8s.io/api/core/v1"
9+
"k8s.io/apimachinery/pkg/types"
10+
"k8s.io/apimachinery/pkg/util/sets"
11+
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
13+
)
14+
15+
type EnhancedBackend struct{}
16+
17+
// EnhancedBackendBuilder is capable of build EnhancedBackend for Service backend.
18+
type EnhancedBackendBuilder interface {
19+
Build(ctx context.Context, svc *corev1.Service, action Action, backendServices map[types.NamespacedName]*corev1.Service) error
20+
}
21+
22+
// NewDefaultEnhancedBackendBuilder constructs new defaultEnhancedBackendBuilder.
23+
func NewDefaultEnhancedBackendBuilder(k8sClient client.Client, annotationParser annotations.Parser, logger logr.Logger) *defaultEnhancedBackendBuilder {
24+
return &defaultEnhancedBackendBuilder{
25+
k8sClient: k8sClient,
26+
annotationParser: annotationParser,
27+
logger: logger,
28+
}
29+
}
30+
31+
type defaultEnhancedBackendBuilder struct {
32+
k8sClient client.Client
33+
annotationParser annotations.Parser
34+
logger logr.Logger
35+
}
36+
37+
func (b *defaultEnhancedBackendBuilder) Build(ctx context.Context, svc *corev1.Service, action Action, backendServices map[types.NamespacedName]*corev1.Service) error {
38+
if err := b.loadBackendServices(ctx, &action, svc.Namespace, backendServices); err != nil {
39+
return err
40+
}
41+
42+
return nil
43+
}
44+
45+
// loadBackendServices loads referenced backend services into backendServices.
46+
func (b *defaultEnhancedBackendBuilder) loadBackendServices(ctx context.Context, action *Action, namespace string,
47+
backendServices map[types.NamespacedName]*corev1.Service) error {
48+
svcNames := sets.NewString()
49+
for _, tgt := range action.ForwardConfig.TargetGroups {
50+
if tgt.ServiceName != nil {
51+
svcNames.Insert(awssdk.ToString(tgt.ServiceName))
52+
}
53+
}
54+
55+
for svcName := range svcNames {
56+
svcKey := types.NamespacedName{Namespace: namespace, Name: svcName}
57+
if _, ok := backendServices[svcKey]; ok {
58+
continue
59+
}
60+
61+
// Fetch the Service from the API
62+
svc := &corev1.Service{}
63+
err := b.k8sClient.Get(ctx, svcKey, svc)
64+
if err != nil {
65+
return err
66+
}
67+
68+
backendServices[svcKey] = svc
69+
}
70+
71+
return nil
72+
}

‎pkg/service/enhanced_backend_builder_test.go‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
package service
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
8+
9+
awssdk "github.com/aws/aws-sdk-go-v2/aws"
10+
"github.com/google/go-cmp/cmp"
11+
"github.com/stretchr/testify/assert"
12+
corev1 "k8s.io/api/core/v1"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/runtime"
15+
"k8s.io/apimachinery/pkg/types"
16+
"k8s.io/apimachinery/pkg/util/intstr"
17+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
18+
"sigs.k8s.io/aws-load-balancer-controller/pkg/equality"
19+
testclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
20+
)
21+
22+
var svc1 = &corev1.Service{
23+
ObjectMeta: metav1.ObjectMeta{
24+
Namespace: "awesome-ns",
25+
Name: "svc-1",
26+
},
27+
}
28+
29+
var svc2 = &corev1.Service{
30+
ObjectMeta: metav1.ObjectMeta{
31+
Namespace: "awesome-ns",
32+
Name: "svc-2",
33+
},
34+
}
35+
36+
func Test_defaultEnhancedBackendBuilder_Build(t *testing.T) {
37+
portTCP := intstr.FromString("tcp")
38+
39+
type env struct {
40+
svcs []*corev1.Service
41+
}
42+
type args struct {
43+
svc *corev1.Service
44+
action Action
45+
backendServices map[types.NamespacedName]*corev1.Service
46+
}
47+
48+
tests := []struct {
49+
name string
50+
env env
51+
args args
52+
want EnhancedBackend
53+
wantBackendServices map[types.NamespacedName]*corev1.Service
54+
wantErr error
55+
}{
56+
{
57+
name: "builds enhanced backend",
58+
env: env{svcs: []*corev1.Service{svc1}},
59+
args: args{
60+
svc: svc1,
61+
action: Action{
62+
Type: ActionTypeForward,
63+
ForwardConfig: &ForwardActionConfig{
64+
TargetGroups: []TargetGroupTuple{
65+
{
66+
ServiceName: awssdk.String("svc-1"),
67+
ServicePort: &portTCP,
68+
},
69+
},
70+
},
71+
},
72+
backendServices: map[types.NamespacedName]*corev1.Service{},
73+
},
74+
wantBackendServices: map[types.NamespacedName]*corev1.Service{
75+
types.NamespacedName{Namespace: "awesome-ns", Name: "svc-1"}: svc1,
76+
},
77+
},
78+
}
79+
for _, tt := range tests {
80+
t.Run(tt.name, func(t *testing.T) {
81+
ctx := context.Background()
82+
k8sSchema := runtime.NewScheme()
83+
clientgoscheme.AddToScheme(k8sSchema)
84+
k8sClient := testclient.NewClientBuilder().WithScheme(k8sSchema).Build()
85+
for _, svc := range tt.env.svcs {
86+
assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy()))
87+
}
88+
annotationParser := annotations.NewSuffixAnnotationParser("service.beta.kubernetes.io")
89+
b := &defaultEnhancedBackendBuilder{
90+
k8sClient: k8sClient,
91+
annotationParser: annotationParser,
92+
}
93+
94+
err := b.Build(context.Background(), tt.args.svc, tt.args.action, tt.args.backendServices)
95+
assert.NoError(t, err)
96+
})
97+
}
98+
}
99+
100+
func Test_defaultEnhancedBackendBuilder_loadBackendServices(t *testing.T) {
101+
port80 := intstr.FromInt(80)
102+
103+
type env struct {
104+
svcs []*corev1.Service
105+
}
106+
type args struct {
107+
action *Action
108+
namespace string
109+
backendServices map[types.NamespacedName]*corev1.Service
110+
}
111+
tests := []struct {
112+
name string
113+
env env
114+
args args
115+
wantAction Action
116+
wantBackendServices map[types.NamespacedName]*corev1.Service
117+
wantErr error
118+
}{
119+
{
120+
name: "forward to a single service",
121+
env: env{
122+
svcs: []*corev1.Service{svc1, svc2},
123+
},
124+
args: args{
125+
action: &Action{
126+
Type: ActionTypeForward,
127+
ForwardConfig: &ForwardActionConfig{
128+
TargetGroups: []TargetGroupTuple{
129+
{
130+
ServiceName: awssdk.String("svc-1"),
131+
ServicePort: &port80,
132+
},
133+
},
134+
},
135+
},
136+
namespace: "awesome-ns",
137+
backendServices: map[types.NamespacedName]*corev1.Service{},
138+
},
139+
wantBackendServices: map[types.NamespacedName]*corev1.Service{
140+
types.NamespacedName{Namespace: "awesome-ns", Name: "svc-1"}: svc1,
141+
},
142+
},
143+
{
144+
name: "forward to multiple services",
145+
env: env{
146+
svcs: []*corev1.Service{svc1, svc2},
147+
},
148+
args: args{
149+
action: &Action{
150+
Type: ActionTypeForward,
151+
ForwardConfig: &ForwardActionConfig{
152+
TargetGroups: []TargetGroupTuple{
153+
{
154+
ServiceName: awssdk.String("svc-1"),
155+
ServicePort: &port80,
156+
},
157+
{
158+
ServiceName: awssdk.String("svc-2"),
159+
ServicePort: &port80,
160+
},
161+
},
162+
},
163+
},
164+
namespace: "awesome-ns",
165+
backendServices: map[types.NamespacedName]*corev1.Service{},
166+
},
167+
wantBackendServices: map[types.NamespacedName]*corev1.Service{
168+
types.NamespacedName{Namespace: "awesome-ns", Name: "svc-1"}: svc1,
169+
types.NamespacedName{Namespace: "awesome-ns", Name: "svc-2"}: svc2,
170+
},
171+
},
172+
}
173+
for _, tt := range tests {
174+
t.Run(tt.name, func(t *testing.T) {
175+
ctx := context.Background()
176+
k8sSchema := runtime.NewScheme()
177+
clientgoscheme.AddToScheme(k8sSchema)
178+
k8sClient := testclient.NewClientBuilder().WithScheme(k8sSchema).Build()
179+
for _, svc := range tt.env.svcs {
180+
assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy()))
181+
}
182+
183+
b := &defaultEnhancedBackendBuilder{
184+
k8sClient: k8sClient,
185+
}
186+
err := b.loadBackendServices(ctx, tt.args.action, tt.args.namespace, tt.args.backendServices)
187+
if tt.wantErr != nil {
188+
assert.EqualError(t, err, tt.wantErr.Error())
189+
} else {
190+
assert.NoError(t, err)
191+
opt := equality.IgnoreFakeClientPopulatedFields()
192+
assert.True(t, cmp.Equal(tt.wantBackendServices, tt.args.backendServices, opt),
193+
"diff: %v", cmp.Diff(tt.wantBackendServices, tt.args.backendServices, opt))
194+
}
195+
})
196+
}
197+
}

0 commit comments

Comments
 (0)