From 0f1e05f07eeeed4fd66ac4e29cf4dcddfc1d0a2b Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Wed, 23 Jul 2025 14:16:34 +0200 Subject: [PATCH 1/2] Add mTLS support to runtime extension server and client --- exp/runtime/server/server.go | 19 +++-- internal/runtime/client/client.go | 14 ++++ internal/runtime/client/client_test.go | 99 ++++++++++++++++++++++++++ main.go | 14 +++- 4 files changed, 137 insertions(+), 9 deletions(-) diff --git a/exp/runtime/server/server.go b/exp/runtime/server/server.go index 540636bc04d3..1252401f3aeb 100644 --- a/exp/runtime/server/server.go +++ b/exp/runtime/server/server.go @@ -80,6 +80,10 @@ type Options struct { // Note: This option is only used when TLSOpts does not set GetCertificate. KeyName string + // ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate. + // Defaults to "", which means server does not verify client's certificate. + ClientCAName string + // TLSOpts is used to allow configuring the TLS config used for the server. // This also allows providing a certificate via GetCertificate. TLSOpts []func(*tls.Config) @@ -105,13 +109,14 @@ func New(options Options) (*Server, error) { webhookServer := webhook.NewServer( webhook.Options{ - Port: options.Port, - Host: options.Host, - CertDir: options.CertDir, - CertName: options.CertName, - KeyName: options.KeyName, - TLSOpts: options.TLSOpts, - WebhookMux: http.NewServeMux(), + Port: options.Port, + Host: options.Host, + ClientCAName: options.ClientCAName, + CertDir: options.CertDir, + CertName: options.CertName, + KeyName: options.KeyName, + TLSOpts: options.TLSOpts, + WebhookMux: http.NewServeMux(), }, ) diff --git a/internal/runtime/client/client.go b/internal/runtime/client/client.go index 3d4619feda93..01c2df137744 100644 --- a/internal/runtime/client/client.go +++ b/internal/runtime/client/client.go @@ -60,6 +60,8 @@ const defaultDiscoveryTimeout = 10 * time.Second // Options are creation options for a Client. type Options struct { + CertFile string // Path of the PEM-encoded client certificate. + KeyFile string // Path of the PEM-encoded client key. Catalog *runtimecatalog.Catalog Registry runtimeregistry.ExtensionRegistry Client ctrlclient.Client @@ -68,6 +70,8 @@ type Options struct { // New returns a new Client. func New(options Options) runtimeclient.Client { return &client{ + certFile: options.CertFile, + keyFile: options.KeyFile, catalog: options.Catalog, registry: options.Registry, client: options.Client, @@ -77,6 +81,8 @@ func New(options Options) runtimeclient.Client { var _ runtimeclient.Client = &client{} type client struct { + certFile string + keyFile string catalog *runtimecatalog.Catalog registry runtimeregistry.ExtensionRegistry client ctrlclient.Client @@ -102,6 +108,8 @@ func (c *client) Discover(ctx context.Context, extensionConfig *runtimev1.Extens request := &runtimehooksv1.DiscoveryRequest{} response := &runtimehooksv1.DiscoveryResponse{} opts := &httpCallOptions{ + certFile: c.certFile, + keyFile: c.keyFile, catalog: c.catalog, config: extensionConfig.Spec.ClientConfig, registrationGVH: hookGVH, @@ -329,6 +337,8 @@ func (c *client) CallExtension(ctx context.Context, hook runtimecatalog.Hook, fo } httpOpts := &httpCallOptions{ + certFile: c.certFile, + keyFile: c.keyFile, catalog: c.catalog, config: registration.ClientConfig, registrationGVH: registration.GroupVersionHook, @@ -396,6 +406,8 @@ func cloneAndAddSettings(request runtimehooksv1.RequestObject, registrationSetti } type httpCallOptions struct { + certFile string + keyFile string catalog *runtimecatalog.Catalog config runtimev1.ClientConfig registrationGVH runtimecatalog.GroupVersionHook @@ -484,6 +496,8 @@ func httpCall(ctx context.Context, request, response runtime.Object, opts *httpC client := http.DefaultClient tlsConfig, err := transport.TLSConfigFor(&transport.Config{ TLS: transport.TLSConfig{ + CertFile: opts.certFile, + KeyFile: opts.keyFile, CAData: opts.config.CABundle, ServerName: extensionURL.Hostname(), }, diff --git a/internal/runtime/client/client_test.go b/internal/runtime/client/client_test.go index 70a300529b24..f457b18d4093 100644 --- a/internal/runtime/client/client_test.go +++ b/internal/runtime/client/client_test.go @@ -19,10 +19,13 @@ package client import ( "context" "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" "regexp" "testing" @@ -842,6 +845,102 @@ func TestClient_CallExtension(t *testing.T) { } } +func TestClient_CallExtensionWithClientAuthentication(t *testing.T) { + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + + validExtensionHandlerWithFailPolicy := runtimev1.ExtensionConfig{ + ObjectMeta: metav1.ObjectMeta{ + ResourceVersion: "15", + }, + Spec: runtimev1.ExtensionConfigSpec{ + ClientConfig: runtimev1.ClientConfig{ + // Set a fake URL, in test cases where we start the test server the URL will be overridden. + URL: "https://127.0.0.1/", + CABundle: testcerts.CACert, + }, + NamespaceSelector: &metav1.LabelSelector{}, + }, + Status: runtimev1.ExtensionConfigStatus{ + Handlers: []runtimev1.ExtensionHandler{ + { + Name: "valid-extension", + RequestHook: runtimev1.GroupVersionHook{ + APIVersion: fakev1alpha1.GroupVersion.String(), + Hook: "FakeHook", + }, + TimeoutSeconds: 1, + FailurePolicy: runtimev1.FailurePolicyFail, + }, + }, + }, + } + + g := NewWithT(t) + + tmpDir := t.TempDir() + clientCertFile := filepath.Join(tmpDir, "tls.crt") + g.Expect(os.WriteFile(clientCertFile, testcerts.ClientCert, 0600)).To(Succeed()) + clientKeyFile := filepath.Join(tmpDir, "tls.key") + g.Expect(os.WriteFile(clientKeyFile, testcerts.ClientKey, 0600)).To(Succeed()) + + var serverCallCount int + srv := createSecureTestServer(testServerConfig{ + start: true, + responses: map[string]testServerResponse{ + "/*": response(runtimehooksv1.ResponseStatusSuccess), + }, + }, func() { + serverCallCount++ + }) + + // Setup the runtime extension server so it requires client authentication with certificates signed by a given CA. + certpool := x509.NewCertPool() + certpool.AppendCertsFromPEM(testcerts.CACert) + srv.TLS.ClientAuth = tls.RequireAndVerifyClientCert + srv.TLS.ClientCAs = certpool + + srv.StartTLS() + defer srv.Close() + + // Set the URL to the real address of the test server. + validExtensionHandlerWithFailPolicy.Spec.ClientConfig.URL = fmt.Sprintf("https://%s/", srv.Listener.Addr().String()) + + cat := runtimecatalog.New() + _ = fakev1alpha1.AddToCatalog(cat) + _ = fakev1alpha2.AddToCatalog(cat) + fakeClient := fake.NewClientBuilder(). + WithObjects(ns). + Build() + + c := New(Options{ + // Add client authentication credentials to the client + CertFile: clientCertFile, + KeyFile: clientKeyFile, + Catalog: cat, + Registry: registry([]runtimev1.ExtensionConfig{validExtensionHandlerWithFailPolicy}), + Client: fakeClient, + }) + + obj := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: "foo", + }, + } + // Call once without caching. + err := c.CallExtension(context.Background(), fakev1alpha1.FakeHook, obj, "valid-extension", &fakev1alpha1.FakeRequest{}, &fakev1alpha1.FakeResponse{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(serverCallCount).To(Equal(1)) +} + func cacheKeyFunc(extensionName, extensionConfigResourceVersion string, request runtimehooksv1.RequestObject) string { // Note: extensionName is identical to the value of the name parameter passed into CallExtension. s := fmt.Sprintf("%s-%s", extensionName, extensionConfigResourceVersion) diff --git a/main.go b/main.go index 6a5e9b69d0de..06a263cef021 100644 --- a/main.go +++ b/main.go @@ -110,6 +110,8 @@ var ( webhookCertDir string webhookCertName string webhookKeyName string + runtimeExtensionCertFile string + runtimeExtensionKeyFile string healthAddr string managerOptions = flags.ManagerOptions{} logOptions = logs.NewOptions() @@ -259,10 +261,16 @@ func InitFlags(fs *pflag.FlagSet) { "Webhook cert dir.") fs.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", - "Webhook cert name.") + "Name of the file for webhook's server certificate; the file must be placed under webhook-cert-dir.") fs.StringVar(&webhookKeyName, "webhook-key-name", "tls.key", - "Webhook key name.") + "Name of the file for webhook's server key; the file must be placed under webhook-cert-dir.") + + fs.StringVar(&runtimeExtensionCertFile, "runtime-extension-client-cert-file", "", + "Path of the PEM-encoded client certificate to be used when calling runtime extensions.") + + fs.StringVar(&runtimeExtensionKeyFile, "runtime-extension-client-key-file", "", + "Path of the PEM-encoded client key to be used when calling runtime extensions.") fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") @@ -531,6 +539,8 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager, watchNamespaces map if feature.Gates.Enabled(feature.RuntimeSDK) { // This is the creation of the runtimeClient for the controllers, embedding a shared catalog and registry instance. runtimeClient = internalruntimeclient.New(internalruntimeclient.Options{ + CertFile: runtimeExtensionCertFile, + KeyFile: runtimeExtensionKeyFile, Catalog: catalog, Registry: runtimeregistry.New(), Client: mgr.GetClient(), From a8cd4f8da13ce062f244d6095519e3d305d1627f Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Wed, 23 Jul 2025 17:24:34 +0200 Subject: [PATCH 2/2] Fix unit test for release-1.10 --- internal/runtime/client/client_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/runtime/client/client_test.go b/internal/runtime/client/client_test.go index f457b18d4093..775754c9df98 100644 --- a/internal/runtime/client/client_test.go +++ b/internal/runtime/client/client_test.go @@ -863,7 +863,7 @@ func TestClient_CallExtensionWithClientAuthentication(t *testing.T) { Spec: runtimev1.ExtensionConfigSpec{ ClientConfig: runtimev1.ClientConfig{ // Set a fake URL, in test cases where we start the test server the URL will be overridden. - URL: "https://127.0.0.1/", + URL: ptr.To("https://127.0.0.1/"), CABundle: testcerts.CACert, }, NamespaceSelector: &metav1.LabelSelector{}, @@ -876,8 +876,8 @@ func TestClient_CallExtensionWithClientAuthentication(t *testing.T) { APIVersion: fakev1alpha1.GroupVersion.String(), Hook: "FakeHook", }, - TimeoutSeconds: 1, - FailurePolicy: runtimev1.FailurePolicyFail, + TimeoutSeconds: ptr.To[int32](1), + FailurePolicy: ptr.To(runtimev1.FailurePolicyFail), }, }, }, @@ -911,7 +911,7 @@ func TestClient_CallExtensionWithClientAuthentication(t *testing.T) { defer srv.Close() // Set the URL to the real address of the test server. - validExtensionHandlerWithFailPolicy.Spec.ClientConfig.URL = fmt.Sprintf("https://%s/", srv.Listener.Addr().String()) + validExtensionHandlerWithFailPolicy.Spec.ClientConfig.URL = ptr.To(fmt.Sprintf("https://%s/", srv.Listener.Addr().String())) cat := runtimecatalog.New() _ = fakev1alpha1.AddToCatalog(cat)