Support serviceaccount pull secrets

Serviceaccounts reference pull secrets!

* Determine our serviceaccount (via the new internal/shared/util/sa package).
* Use a common pull_secret_controller
* Update the pull_secret_controller to know about the service account
* Update the pull_secret_controller to watch the namespace-local secrets
* Update caching to include sa, and use filters for additional secrets
* Add RBAC to access these secrets and sa
* Update writing the auth.json file to handle dockercfg and dockerconfigjson
* Update writing the auth.json file to include multiple secrets

Signed-off-by: Todd Short <[email protected]>

Author: tmshort

cmd/catalogd/main.go

@@ -61,8 +61,10 @@ import (
 	"github.com/operator-framework/operator-controller/internal/catalogd/serverutil"
 	"github.com/operator-framework/operator-controller/internal/catalogd/storage"
 	"github.com/operator-framework/operator-controller/internal/catalogd/webhook"
+	sharedcontrollers "github.com/operator-framework/operator-controller/internal/shared/controllers"
 	fsutil "github.com/operator-framework/operator-controller/internal/shared/util/fs"
 	imageutil "github.com/operator-framework/operator-controller/internal/shared/util/image"
+	sautil "github.com/operator-framework/operator-controller/internal/shared/util/sa"
 	"github.com/operator-framework/operator-controller/internal/shared/version"
 )
 
@@ -246,18 +248,40 @@ func run(ctx context.Context) error {
 	cacheOptions := crcache.Options{
 		ByObject: map[client.Object]crcache.ByObject{},
 	}
-	if cfg.globalPullSecretKey != nil {
-		cacheOptions.ByObject[&corev1.Secret{}] = crcache.ByObject{
-			Namespaces: map[string]crcache.Config{
-				cfg.globalPullSecretKey.Namespace: {
-					LabelSelector: k8slabels.Everything(),
-					FieldSelector: fields.SelectorFromSet(map[string]string{
-						"metadata.name": cfg.globalPullSecretKey.Name,
-					}),
-				},
+
+	saKey, err := sautil.GetServiceAccount()
+	if err != nil {
+		setupLog.Error(err, "Unable to get pod namesapce and serviceaccount")
+		return err
+	}
+
+	setupLog.Info("Read token", "serviceaccount", saKey)
+	cacheOptions.ByObject[&corev1.ServiceAccount{}] = crcache.ByObject{
+		Namespaces: map[string]crcache.Config{
+			saKey.Namespace: {
+				LabelSelector: k8slabels.Everything(),
+				FieldSelector: fields.SelectorFromSet(map[string]string{
+					"metadata.name": saKey.Name,
+				}),
 			},
+		},
+	}
+
+	secretCache := crcache.ByObject{}
+	secretCache.Namespaces = make(map[string]crcache.Config, 2)
+	secretCache.Namespaces[saKey.Namespace] = crcache.Config{
+		LabelSelector: k8slabels.Everything(),
+		FieldSelector: fields.Everything(),
+	}
+	if cfg.globalPullSecretKey != nil {
+		secretCache.Namespaces[cfg.globalPullSecretKey.Namespace] = crcache.Config{
+			LabelSelector: k8slabels.Everything(),
+			FieldSelector: fields.SelectorFromSet(map[string]string{
+				"metadata.name": cfg.globalPullSecretKey.Name,
+			}),
 		}
 	}
+	cacheOptions.ByObject[&corev1.Secret{}] = secretCache
 
 	// Create manager
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -312,7 +336,7 @@ func run(ctx context.Context) error {
 				DockerCertPath: cfg.pullCasDir,
 				OCICertPath:    cfg.pullCasDir,
 			}
-			if _, err := os.Stat(authFilePath); err == nil && cfg.globalPullSecretKey != nil {
+			if _, err := os.Stat(authFilePath); err == nil {
 				logger.Info("using available authentication information for pulling image")
 				srcContext.AuthFilePath = authFilePath
 			} else if os.IsNotExist(err) {
@@ -370,17 +394,16 @@ func run(ctx context.Context) error {
 		return err
 	}
 
-	if cfg.globalPullSecretKey != nil {
-		setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", cfg.globalPullSecret)
-		err := (&corecontrollers.PullSecretReconciler{
-			Client:       mgr.GetClient(),
-			AuthFilePath: authFilePath,
-			SecretKey:    *cfg.globalPullSecretKey,
-		}).SetupWithManager(mgr)
-		if err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer")
-			return err
-		}
+	setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", cfg.globalPullSecret)
+	err = (&sharedcontrollers.PullSecretReconciler{
+		Client:            mgr.GetClient(),
+		AuthFilePath:      authFilePath,
+		SecretKey:         cfg.globalPullSecretKey,
+		ServiceAccountKey: saKey,
+	}).SetupWithManager(mgr)
+	if err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer")
+		return err
 	}
 	//+kubebuilder:scaffold:builder
 

cmd/operator-controller/main.go

@@ -71,9 +71,11 @@ import (
 	"github.com/operator-framework/operator-controller/internal/operator-controller/rukpak/render/certproviders"
 	"github.com/operator-framework/operator-controller/internal/operator-controller/rukpak/render/registryv1"
 	"github.com/operator-framework/operator-controller/internal/operator-controller/scheme"
+	sharedcontrollers "github.com/operator-framework/operator-controller/internal/shared/controllers"
 	fsutil "github.com/operator-framework/operator-controller/internal/shared/util/fs"
 	httputil "github.com/operator-framework/operator-controller/internal/shared/util/http"
 	imageutil "github.com/operator-framework/operator-controller/internal/shared/util/image"
+	sautil "github.com/operator-framework/operator-controller/internal/shared/util/sa"
 	"github.com/operator-framework/operator-controller/internal/shared/version"
 )
 
@@ -217,18 +219,40 @@ func run() error {
 		},
 		DefaultLabelSelector: k8slabels.Nothing(),
 	}
-	if globalPullSecretKey != nil {
-		cacheOptions.ByObject[&corev1.Secret{}] = crcache.ByObject{
-			Namespaces: map[string]crcache.Config{
-				globalPullSecretKey.Namespace: {
-					LabelSelector: k8slabels.Everything(),
-					FieldSelector: fields.SelectorFromSet(map[string]string{
-						"metadata.name": globalPullSecretKey.Name,
-					}),
-				},
+
+	saKey, err := sautil.GetServiceAccount()
+	if err != nil {
+		setupLog.Error(err, "Unable to get pod namesapce and serviceaccount")
+		return err
+	}
+
+	setupLog.Info("Read token", "serviceaccount", saKey)
+	cacheOptions.ByObject[&corev1.ServiceAccount{}] = crcache.ByObject{
+		Namespaces: map[string]crcache.Config{
+			saKey.Namespace: {
+				LabelSelector: k8slabels.Everything(),
+				FieldSelector: fields.SelectorFromSet(map[string]string{
+					"metadata.name": saKey.Name,
+				}),
 			},
+		},
+	}
+
+	secretCache := crcache.ByObject{}
+	secretCache.Namespaces = make(map[string]crcache.Config, 2)
+	secretCache.Namespaces[saKey.Namespace] = crcache.Config{
+		LabelSelector: k8slabels.Everything(),
+		FieldSelector: fields.Everything(),
+	}
+	if globalPullSecretKey != nil {
+		secretCache.Namespaces[globalPullSecretKey.Namespace] = crcache.Config{
+			LabelSelector: k8slabels.Everything(),
+			FieldSelector: fields.SelectorFromSet(map[string]string{
+				"metadata.name": globalPullSecretKey.Name,
+			}),
 		}
 	}
+	cacheOptions.ByObject[&corev1.Secret{}] = secretCache
 
 	metricsServerOptions := server.Options{}
 	if len(cfg.certFile) > 0 && len(cfg.keyFile) > 0 {
@@ -360,7 +384,7 @@ func run() error {
 				OCICertPath:    cfg.pullCasDir,
 			}
 			logger := log.FromContext(ctx)
-			if _, err := os.Stat(authFilePath); err == nil && globalPullSecretKey != nil {
+			if _, err := os.Stat(authFilePath); err == nil {
 				logger.Info("using available authentication information for pulling image")
 				srcContext.AuthFilePath = authFilePath
 			} else if os.IsNotExist(err) {
@@ -482,17 +506,16 @@ func run() error {
 		return err
 	}
 
-	if globalPullSecretKey != nil {
-		setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", cfg.globalPullSecret)
-		err := (&controllers.PullSecretReconciler{
-			Client:       mgr.GetClient(),
-			AuthFilePath: authFilePath,
-			SecretKey:    *globalPullSecretKey,
-		}).SetupWithManager(mgr)
-		if err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer")
-			return err
-		}
+	setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", cfg.globalPullSecret)
+	err = (&sharedcontrollers.PullSecretReconciler{
+		Client:            mgr.GetClient(),
+		AuthFilePath:      authFilePath,
+		SecretKey:         globalPullSecretKey,
+		ServiceAccountKey: saKey,
+	}).SetupWithManager(mgr)
+	if err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer")
+		return err
 	}
 
 	//+kubebuilder:scaffold:builder

config/base/catalogd/rbac/role.yaml

@@ -4,6 +4,14 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - olm.operatorframework.io
   resources:
@@ -30,3 +38,18 @@ rules:
   - get
   - patch
   - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - list
+  - watch

config/base/catalogd/rbac/role_binding.yaml

@@ -13,3 +13,20 @@ subjects:
 - kind: ServiceAccount
   name: controller-manager
   namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/part-of: olm
+    app.kubernetes.io/name: catalogd
+  name: manager-rolebinding
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system

config/base/operator-controller/rbac/role.yaml

@@ -77,3 +77,11 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - list
+  - watch

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/containers/image/v5 v5.35.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-logr/logr v1.4.3
+	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-containerregistry v0.20.3
 	github.com/gorilla/handlers v1.5.2

go.sum

@@ -209,6 +209,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-migrate/migrate/v4 v4.18.3 h1:EYGkoOsvgHHfm5U/naS1RP/6PL/Xv3S4B/swMiAmDLs=
 github.com/golang-migrate/migrate/v4 v4.18.3/go.mod h1:99BKpIi6ruaaXRM1A77eqZ+FWPQ3cfRa+ZVy5bmWMaY=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

internal/catalogd/controllers/core/clustercatalog_controller.go

@@ -79,6 +79,8 @@ type storedCatalogData struct {
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clustercatalogs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clustercatalogs/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clustercatalogs/finalizers,verbs=update
+//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
+//+kubebuilder:rbac:namespace=system,groups=core,resources=serviceaccounts,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.