diff --git a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml
index cc5275d0e..c41580a13 100644
--- a/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml
+++ b/bundle/manifests/monitoring.rhobs_thanosqueriers.yaml
@@ -110,6 +110,67 @@ spec:
type: object
type: object
x-kubernetes-map-type: atomic
+ webTLSConfig:
+ description: Configure TLS options for the Thanos web server.
+ properties:
+ certificate:
+ description: Reference to the TLS public certificate for the web
+ server.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ certificateAuthority:
+ description: Reference to the root Certificate Authority used
+ to verify the web server's certificate.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ privateKey:
+ description: Reference to the TLS private key for the web server.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ required:
+ - certificate
+ - certificateAuthority
+ - privateKey
+ type: object
required:
- selector
type: object
diff --git a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml
index d708ea933..e64f4ded5 100644
--- a/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml
+++ b/deploy/crds/common/monitoring.rhobs_thanosqueriers.yaml
@@ -110,6 +110,67 @@ spec:
type: object
type: object
x-kubernetes-map-type: atomic
+ webTLSConfig:
+ description: Configure TLS options for the Thanos web server.
+ properties:
+ certificate:
+ description: Reference to the TLS public certificate for the web
+ server.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ certificateAuthority:
+ description: Reference to the root Certificate Authority used
+ to verify the web server's certificate.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ privateKey:
+ description: Reference to the TLS private key for the web server.
+ properties:
+ key:
+ description: The key of the secret to select from. Must be
+ a valid secret key.
+ minLength: 1
+ type: string
+ name:
+ description: The name of the secret in the object's namespace
+ to select from.
+ minLength: 1
+ type: string
+ required:
+ - key
+ - name
+ type: object
+ required:
+ - certificate
+ - certificateAuthority
+ - privateKey
+ type: object
required:
- selector
type: object
diff --git a/docs/api.md b/docs/api.md
index 9b51d3587..51a8419cb 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -3690,6 +3690,13 @@ deduplicate.
false |
+
+ | webTLSConfig |
+ object |
+
+ Configure TLS options for the Thanos web server.
+ |
+ false |
@@ -3810,6 +3817,149 @@ list restricting them.
+
+### ThanosQuerier.spec.webTLSConfig
+[↩ Parent](#thanosquerierspec)
+
+
+
+Configure TLS options for the Thanos web server.
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | certificate |
+ object |
+
+ Reference to the TLS public certificate for the web server.
+ |
+ true |
+
+ | certificateAuthority |
+ object |
+
+ Reference to the root Certificate Authority used to verify the web server's certificate.
+ |
+ true |
+
+ | privateKey |
+ object |
+
+ Reference to the TLS private key for the web server.
+ |
+ true |
+
+
+
+
+### ThanosQuerier.spec.webTLSConfig.certificate
+[↩ Parent](#thanosquerierspecwebtlsconfig)
+
+
+
+Reference to the TLS public certificate for the web server.
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | key |
+ string |
+
+ The key of the secret to select from. Must be a valid secret key.
+ |
+ true |
+
+ | name |
+ string |
+
+ The name of the secret in the object's namespace to select from.
+ |
+ true |
+
+
+
+
+### ThanosQuerier.spec.webTLSConfig.certificateAuthority
+[↩ Parent](#thanosquerierspecwebtlsconfig)
+
+
+
+Reference to the root Certificate Authority used to verify the web server's certificate.
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | key |
+ string |
+
+ The key of the secret to select from. Must be a valid secret key.
+ |
+ true |
+
+ | name |
+ string |
+
+ The name of the secret in the object's namespace to select from.
+ |
+ true |
+
+
+
+
+### ThanosQuerier.spec.webTLSConfig.privateKey
+[↩ Parent](#thanosquerierspecwebtlsconfig)
+
+
+
+Reference to the TLS private key for the web server.
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | key |
+ string |
+
+ The key of the secret to select from. Must be a valid secret key.
+ |
+ true |
+
+ | name |
+ string |
+
+ The name of the secret in the object's namespace to select from.
+ |
+ true |
+
+
+
# observability.openshift.io/v1alpha1
Resource Types:
diff --git a/pkg/apis/monitoring/v1alpha1/types.go b/pkg/apis/monitoring/v1alpha1/types.go
index 6ed6205a2..18eb00df6 100644
--- a/pkg/apis/monitoring/v1alpha1/types.go
+++ b/pkg/apis/monitoring/v1alpha1/types.go
@@ -279,6 +279,9 @@ type ThanosQuerierSpec struct {
// Selector to select which namespaces the Monitoring Stack objects are discovered from.
NamespaceSelector NamespaceSelector `json:"namespaceSelector,omitempty"`
ReplicaLabels []string `json:"replicaLabels,omitempty"`
+ // Configure TLS options for the Thanos web server.
+ // +optional
+ WebTLSConfig *WebTLSConfig `json:"webTLSConfig,omitempty"`
}
// ThanosQuerierStatus defines the observed state of ThanosQuerier.
diff --git a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go
index ca88fb311..c62674f8d 100644
--- a/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go
@@ -348,6 +348,11 @@ func (in *ThanosQuerierSpec) DeepCopyInto(out *ThanosQuerierSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
+ if in.WebTLSConfig != nil {
+ in, out := &in.WebTLSConfig, &out.WebTLSConfig
+ *out = new(WebTLSConfig)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ThanosQuerierSpec.
diff --git a/pkg/controllers/monitoring/thanos-querier/components.go b/pkg/controllers/monitoring/thanos-querier/components.go
index 905539953..160684a78 100644
--- a/pkg/controllers/monitoring/thanos-querier/components.go
+++ b/pkg/controllers/monitoring/thanos-querier/components.go
@@ -13,17 +13,54 @@ import (
"github.com/rhobs/observability-operator/pkg/reconciler"
)
-func thanosComponentReconcilers(thanos *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) []reconciler.Reconciler {
+func thanosComponentReconcilers(
+ thanos *msoapi.ThanosQuerier,
+ sidecarUrls []string,
+ thanosCfg ThanosConfiguration,
+ tlsHashes map[string]string,
+) []reconciler.Reconciler {
name := "thanos-querier-" + thanos.Name
return []reconciler.Reconciler{
reconciler.NewUpdater(newServiceAccount(name, thanos.Namespace), thanos),
- reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg), thanos),
+ reconciler.NewUpdater(newThanosQuerierDeployment(name, thanos, sidecarUrls, thanosCfg, tlsHashes), thanos),
reconciler.NewUpdater(newService(name, thanos.Namespace), thanos),
- reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace), thanos),
+ reconciler.NewUpdater(newServiceMonitor(name, thanos.Namespace, thanos), thanos),
+ reconciler.NewOptionalUpdater(newHttpConfConfigMap(name, thanos), thanos, thanos.Spec.WebTLSConfig != nil),
}
}
-func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecarUrls []string, thanosCfg ThanosConfiguration) *appsv1.Deployment {
+func newHttpConfConfigMap(name string, thanos *msoapi.ThanosQuerier) *corev1.ConfigMap {
+ httpConf := &corev1.ConfigMap{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: corev1.SchemeGroupVersion.String(),
+ Kind: "ConfigMap",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: fmt.Sprintf("%s-http-conf", name),
+ Namespace: thanos.Namespace,
+ },
+ }
+ if thanos.Spec.WebTLSConfig != nil {
+ httpConf.Data = map[string]string{
+ "http.conf": `
+tls_server_config:
+ cert_file: /etc/thanos/tls-assets/web-cert-secret/` + thanos.Spec.WebTLSConfig.Certificate.Key + `
+ key_file: /etc/thanos/tls-assets/web-key-secret/` + thanos.Spec.WebTLSConfig.PrivateKey.Key,
+ }
+ }
+
+ return httpConf
+}
+
+func newThanosQuerierDeployment(
+ name string,
+ spec *msoapi.ThanosQuerier,
+ sidecarUrls []string,
+ thanosCfg ThanosConfiguration,
+ tlsHashes map[string]string,
+) *appsv1.Deployment {
+ httpConfCMName := fmt.Sprintf("%s-http-conf", name)
+
args := []string{
"query",
"--log.format=logfmt",
@@ -38,6 +75,10 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar
args = append(args, fmt.Sprintf("--query.replica-label=%s", rl))
}
+ if spec.Spec.WebTLSConfig != nil {
+ args = append(args, "--http.config=/etc/thanos/tls-assets/web-http-conf-cm/http.conf")
+ }
+
thanos := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: appsv1.SchemeGroupVersion.String(),
@@ -102,6 +143,58 @@ func newThanosQuerierDeployment(name string, spec *msoapi.ThanosQuerier, sidecar
ProgressDeadlineSeconds: ptr.To(int32(300)),
},
}
+ if spec.Spec.WebTLSConfig != nil {
+ thanos.Spec.Template.Spec.Volumes = append(thanos.Spec.Template.Spec.Volumes, []corev1.Volume{
+ {
+ Name: "thanos-web-tls-key",
+ VolumeSource: corev1.VolumeSource{
+ Secret: &corev1.SecretVolumeSource{
+ SecretName: spec.Spec.WebTLSConfig.PrivateKey.Name,
+ },
+ },
+ },
+ {
+ Name: "thanos-web-tls-cert",
+ VolumeSource: corev1.VolumeSource{
+ Secret: &corev1.SecretVolumeSource{
+ SecretName: spec.Spec.WebTLSConfig.Certificate.Name,
+ },
+ },
+ },
+ {
+ Name: "thanos-web-http-conf",
+ VolumeSource: corev1.VolumeSource{
+ ConfigMap: &corev1.ConfigMapVolumeSource{
+ LocalObjectReference: corev1.LocalObjectReference{
+ Name: httpConfCMName,
+ },
+ },
+ },
+ },
+ }...)
+ thanos.Spec.Template.Spec.Containers[0].VolumeMounts = append(thanos.Spec.Template.Spec.Containers[0].VolumeMounts, []corev1.VolumeMount{
+ {
+ Name: "thanos-web-tls-key",
+ MountPath: "/etc/thanos/tls-assets/web-cert-secret",
+ ReadOnly: true,
+ },
+ {
+ Name: "thanos-web-tls-cert",
+ MountPath: "/etc/thanos/tls-assets/web-key-secret",
+ ReadOnly: true,
+ },
+ {
+ Name: "thanos-web-http-conf",
+ MountPath: "/etc/thanos/tls-assets/web-http-conf-cm",
+ ReadOnly: true,
+ },
+ }...)
+ tlsAnnotations := map[string]string{}
+ for name, hash := range tlsHashes {
+ tlsAnnotations[fmt.Sprintf("monitoring.openshift.io/%s-hash", name)] = hash
+ }
+ thanos.ObjectMeta.Annotations = tlsAnnotations
+ }
return thanos
}
@@ -144,8 +237,8 @@ func newService(name string, namespace string) *corev1.Service {
}
}
-func newServiceMonitor(name string, namespace string) *monv1.ServiceMonitor {
- return &monv1.ServiceMonitor{
+func newServiceMonitor(name string, namespace string, thanos *msoapi.ThanosQuerier) *monv1.ServiceMonitor {
+ serviceMonitor := &monv1.ServiceMonitor{
TypeMeta: metav1.TypeMeta{
APIVersion: monv1.SchemeGroupVersion.String(),
Kind: "ServiceMonitor",
@@ -169,6 +262,23 @@ func newServiceMonitor(name string, namespace string) *monv1.ServiceMonitor {
},
},
}
+ if thanos.Spec.WebTLSConfig != nil {
+ serviceMonitor.Spec.Endpoints[0].Scheme = "https"
+ serviceMonitor.Spec.Endpoints[0].TLSConfig = &monv1.TLSConfig{
+ SafeTLSConfig: monv1.SafeTLSConfig{
+ CA: monv1.SecretOrConfigMap{
+ Secret: &corev1.SecretKeySelector{
+ LocalObjectReference: corev1.LocalObjectReference{
+ Name: thanos.Spec.WebTLSConfig.CertificateAuthority.Name,
+ },
+ Key: thanos.Spec.WebTLSConfig.CertificateAuthority.Key,
+ },
+ },
+ ServerName: ptr.To(name),
+ },
+ }
+ }
+ return serviceMonitor
}
func componentLabels(querierName string) map[string]string {
diff --git a/pkg/controllers/monitoring/thanos-querier/controller.go b/pkg/controllers/monitoring/thanos-querier/controller.go
index 87275a996..8ee977a68 100644
--- a/pkg/controllers/monitoring/thanos-querier/controller.go
+++ b/pkg/controllers/monitoring/thanos-querier/controller.go
@@ -14,6 +14,7 @@ package thanos_querier
import (
"context"
+ "crypto/sha256"
"fmt"
"time"
@@ -22,9 +23,11 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/rand"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -51,6 +54,12 @@ type Options struct {
Thanos ThanosConfiguration
}
+const (
+ thanosTLSPrivateKeySecretNameField = ".spec.webTLSConfig.privateKey.name"
+ thanosTLSCertificateSecretNameField = ".spec.webTLSConfig.certificate.name"
+ thanosTLSCertificateAuthoritySecretNameField = ".spec.webTLSConfig.certificateAuthority.name"
+)
+
// RBAC for watching monitoring stacks
//+kubebuilder:rbac:groups=monitoring.rhobs,resources=monitoringstacks,verbs=list;watch
@@ -63,7 +72,8 @@ type Options struct {
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=list;watch;create;update;patch;delete
// RBAC for managing core resources
-//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts,verbs=list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=services;serviceaccounts;configmaps,verbs=list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=secrets,verbs=list;watch
// RBAC for managing Prometheus Operator CRs
//+kubebuilder:rbac:groups=monitoring.rhobs,resources=servicemonitors,verbs=list;watch;create;update;patch;delete
@@ -79,17 +89,56 @@ func RegisterWithManager(mgr ctrl.Manager, opts Options) error {
thanos: opts.Thanos,
}
+ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSPrivateKeySecretNameField, func(rawObj client.Object) []string {
+ // Extract the secret name from the spec, if one is provided
+ cr := rawObj.(*msoapi.ThanosQuerier)
+ if cr.Spec.WebTLSConfig == nil {
+ return nil
+ }
+ return []string{cr.Spec.WebTLSConfig.PrivateKey.Name}
+ }); err != nil {
+ return err
+ }
+
+ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateSecretNameField, func(rawObj client.Object) []string {
+ // Extract the secret name from the spec, if one is provided
+ cr := rawObj.(*msoapi.ThanosQuerier)
+ if cr.Spec.WebTLSConfig == nil {
+ return nil
+ }
+ return []string{cr.Spec.WebTLSConfig.Certificate.Name}
+ }); err != nil {
+ return err
+ }
+
+ if err := mgr.GetFieldIndexer().IndexField(context.Background(), &msoapi.ThanosQuerier{}, thanosTLSCertificateAuthoritySecretNameField, func(rawObj client.Object) []string {
+ // Extract the secret name from the spec, if one is provided
+ cr := rawObj.(*msoapi.ThanosQuerier)
+ if cr.Spec.WebTLSConfig == nil {
+ return nil
+ }
+ return []string{cr.Spec.WebTLSConfig.CertificateAuthority.Name}
+ }); err != nil {
+ return err
+ }
+
p := predicate.GenerationChangedPredicate{}
return ctrl.NewControllerManagedBy(mgr).
For(&msoapi.ThanosQuerier{}).
Owns(&appsv1.Deployment{}).WithEventFilter(p).
Owns(&corev1.ServiceAccount{}).WithEventFilter(p).
Owns(&corev1.Service{}).WithEventFilter(p).
+ Owns(&corev1.ConfigMap{}).WithEventFilter(p).
Watches(
&msoapi.MonitoringStack{},
handler.EnqueueRequestsFromMapFunc(rm.findQueriersForMonitoringStack),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
).
+ Watches(
+ &corev1.Secret{},
+ handler.EnqueueRequestsFromMapFunc(rm.findQueriersForTLSSecrets),
+ builder.WithPredicates(predicate.GenerationChangedPredicate{}),
+ ).
Complete(rm)
}
@@ -113,7 +162,23 @@ func (rm resourceManager) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
return ctrl.Result{RequeueAfter: 10 * time.Second}, err
}
- reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos)
+ tlsHashes := map[string]string{}
+ if querier.Spec.WebTLSConfig != nil {
+ secretSelectors := []msoapi.SecretKeySelector{
+ querier.Spec.WebTLSConfig.CertificateAuthority,
+ querier.Spec.WebTLSConfig.Certificate,
+ querier.Spec.WebTLSConfig.PrivateKey,
+ }
+ for _, secretSelector := range secretSelectors {
+ hash, err := rm.hashOfTLSSecret(secretSelector, querier.Namespace)
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+ tlsHashes[fmt.Sprintf("%s-%s", secretSelector.Name, secretSelector.Key)] = hash
+ }
+ }
+
+ reconcilers := thanosComponentReconcilers(querier, sidecarServices, rm.thanos, tlsHashes)
for _, reconciler := range reconcilers {
err := reconciler.Reconcile(ctx, rm, rm.scheme)
// handle creation / updation errors that can happen due to a stale cache by
@@ -156,6 +221,20 @@ func (rm resourceManager) findSidecarServices(ctx context.Context, tQuerier *mso
return sidecarUrls, nil
}
+func (rm resourceManager) hashOfTLSSecret(selector msoapi.SecretKeySelector, namespace string) (string, error) {
+ var secret corev1.Secret
+ err := rm.Get(context.Background(), types.NamespacedName{
+ Name: selector.Name,
+ Namespace: namespace,
+ }, &secret)
+ if err != nil {
+ return "", fmt.Errorf("Couldn't get TLS secret %s: %s", selector.Name, err)
+ }
+
+ hash := sha256.Sum256(secret.Data[selector.Key])
+ return rand.SafeEncodeString(fmt.Sprint(hash)), nil
+}
+
// Given a Service object, return a url to use as value for --store/--endpoint.
func getEndpointUrl(serviceName string, namespace string) string {
return fmt.Sprintf("dnssrv+_grpc._tcp.%s.%s.svc.cluster.local", serviceName, namespace)
@@ -191,3 +270,41 @@ func (rm resourceManager) findQueriersForMonitoringStack(ctx context.Context, ms
}
return requests
}
+
+// Find all ThanosQueriers, whose TLS secrets fit the given Secret and
+// return a list of reconcile requests, one for each ThanosQuerier.
+func (rm resourceManager) findQueriersForTLSSecrets(ctx context.Context, src client.Object) []reconcile.Request {
+ requests := []reconcile.Request{}
+
+ thanosWatchFields := []string{
+ thanosTLSCertificateAuthoritySecretNameField,
+ thanosTLSCertificateSecretNameField,
+ thanosTLSPrivateKeySecretNameField,
+ }
+
+ for _, field := range thanosWatchFields {
+ crList := &msoapi.ThanosQuerierList{}
+ listOps := &client.ListOptions{
+ FieldSelector: fields.OneTermEqualSelector(field, src.GetName()),
+ Namespace: src.GetNamespace(),
+ }
+ err := rm.Client.List(ctx, crList, listOps)
+ if err != nil {
+ rm.logger.Error(err, "Failed to list Thanosqueriers")
+ return []reconcile.Request{}
+ }
+
+ for _, item := range crList.Items {
+ logger := rm.logger.WithValues("Secret", src.GetNamespace()+"/"+src.GetName())
+ logger.Info("Found a querier whose secret changed, scheduling sync")
+ requests = append(requests, reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Name: item.GetName(),
+ Namespace: item.GetNamespace(),
+ },
+ })
+ }
+ }
+
+ return requests
+}
diff --git a/test/e2e/framework/assertions.go b/test/e2e/framework/assertions.go
index 064953e26..8ca5be455 100644
--- a/test/e2e/framework/assertions.go
+++ b/test/e2e/framework/assertions.go
@@ -172,6 +172,33 @@ func (f *Framework) AssertDeploymentReady(name, namespace string, fns ...OptionF
}
}
+// AssertDeploymentReadyAndStable asserts that a deployment has the desired number of pods running for 2 consecutive polls 5 seconds appart
+func (f *Framework) AssertDeploymentReadyAndStable(name, namespace string, fns ...OptionFn) func(t *testing.T) {
+ option := AssertOption{
+ PollInterval: 5 * time.Second,
+ WaitTimeout: DefaultTestTimeout,
+ }
+ for _, fn := range fns {
+ fn(&option)
+ }
+ return func(t *testing.T) {
+ key := types.NamespacedName{Name: name, Namespace: namespace}
+ if err := wait.PollUntilContextTimeout(context.Background(), option.PollInterval, option.WaitTimeout, true, func(ctx context.Context) (bool, error) {
+ deployment := &appsv1.Deployment{}
+ err := f.K8sClient.Get(context.Background(), key, deployment)
+ if err == nil && deployment.Status.ReadyReplicas == *deployment.Spec.Replicas {
+ time.Sleep(5 * time.Second)
+ err := f.K8sClient.Get(context.Background(), key, deployment)
+ return err == nil && deployment.Status.ReadyReplicas == *deployment.Spec.Replicas, nil
+ } else {
+ return false, nil
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
func (f *Framework) GetResourceWithRetry(t *testing.T, name, namespace string, obj client.Object) {
option := AssertOption{
PollInterval: 5 * time.Second,
diff --git a/test/e2e/thanos_querier_controller_test.go b/test/e2e/thanos_querier_controller_test.go
index f42255e09..b590eb141 100644
--- a/test/e2e/thanos_querier_controller_test.go
+++ b/test/e2e/thanos_querier_controller_test.go
@@ -3,6 +3,8 @@ package e2e
import (
"context"
"fmt"
+ "net"
+ "strings"
"testing"
"time"
@@ -13,6 +15,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/client-go/util/cert"
"sigs.k8s.io/controller-runtime/pkg/client"
msov1 "github.com/rhobs/observability-operator/pkg/apis/monitoring/v1alpha1"
@@ -31,6 +34,10 @@ func TestThanosQuerierController(t *testing.T) {
name: "Delete resources if matched monitoring stack is deleted",
scenario: stackWithSidecarGetsDeleted,
},
+ {
+ name: "Create resources for single monitoring stack with web endpoint TLS",
+ scenario: singleStackWithSidecarTLS,
+ },
}
for _, tc := range ts {
@@ -115,6 +122,113 @@ func singleStackWithSidecar(t *testing.T) {
}
}
+func singleStackWithSidecarTLS(t *testing.T) {
+ comboName := "tq-ms-combo-tls"
+ querierName := "thanos-querier-" + comboName
+
+ certs, key, err := cert.GenerateSelfSignedCertKey(querierName, []net.IP{}, []string{})
+ assert.NilError(t, err)
+
+ thanosKey := string(key)
+ thanosCerts := strings.SplitAfter(string(certs), "-----END CERTIFICATE-----")
+
+ tlsSecretName := "thanos-test-tls-secret"
+
+ thanosTLSSecret := corev1.Secret{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: corev1.SchemeGroupVersion.String(),
+ Kind: "Secret",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: tlsSecretName,
+ Namespace: e2eTestNamespace,
+ },
+ StringData: map[string]string{
+ "tls.key": thanosKey,
+ "tls.crt": thanosCerts[0],
+ "ca.crt": thanosCerts[1],
+ },
+ }
+ err = f.K8sClient.Create(context.Background(), &thanosTLSSecret)
+ assert.NilError(t, err)
+
+ tq, ms := newThanosStackCombo(t, comboName)
+ tq.Spec.WebTLSConfig = &msov1.WebTLSConfig{
+ PrivateKey: msov1.SecretKeySelector{
+ Name: tlsSecretName,
+ Key: "tls.key",
+ },
+ Certificate: msov1.SecretKeySelector{
+ Name: tlsSecretName,
+ Key: "tls.crt",
+ },
+ CertificateAuthority: msov1.SecretKeySelector{
+ Name: tlsSecretName,
+ Key: "ca.crt",
+ },
+ }
+ err = f.K8sClient.Create(context.Background(), tq)
+ assert.NilError(t, err, "failed to create a thanos querier")
+ err = f.K8sClient.Create(context.Background(), ms)
+ assert.NilError(t, err, "failed to create a monitoring stack")
+
+ // Creating a basic combo must create a thanos deployment and a service
+ thanosDeployment := appsv1.Deployment{}
+ f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosDeployment)
+
+ thanosService := corev1.Service{}
+ f.GetResourceWithRetry(t, querierName, tq.Namespace, &thanosService)
+
+ f.AssertDeploymentReadyAndStable(querierName, tq.Namespace, framework.WithTimeout(5*time.Minute))(t)
+
+ // Assert prometheus instance can be queried
+ stopChan := make(chan struct{})
+ defer close(stopChan)
+ if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+ err = f.StartServicePortForward(querierName, e2eTestNamespace, "10902", stopChan)
+ return err == nil, nil
+ }); wait.Interrupted(err) {
+ t.Fatal("timeout waiting for port-forward")
+ }
+
+ promClient, err := framework.NewTLSPrometheusClient("https://localhost:10902", thanosCerts[1], querierName)
+ if err != nil {
+ t.Fatal(fmt.Errorf("Failed to create prometheus client: %s", err))
+ }
+ expectedResults := map[string]int{
+ "prometheus_build_info": 2, // must return from both prometheus pods
+ }
+ var lastErr error
+ if err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+ correct := 0
+ for query, value := range expectedResults {
+ result, err := promClient.Query(query)
+ if err != nil {
+ return false, nil
+ }
+
+ if len(result.Data.Result) == 0 {
+ return false, nil
+ }
+
+ if len(result.Data.Result) > value {
+ lastErr = fmt.Errorf("invalid result for query %s, got %d, want %d", query, len(result.Data.Result), value)
+ return true, lastErr
+ }
+
+ if len(result.Data.Result) != value {
+ return false, nil
+ }
+
+ correct++
+ }
+
+ return correct == len(expectedResults), nil
+ }); wait.Interrupted(err) {
+ t.Fatal(fmt.Errorf("querying thanos did not yield expected results: %w", lastErr))
+ }
+}
+
func newThanosQuerier(t *testing.T, name string, selector map[string]string) *msov1.ThanosQuerier {
tq := &msov1.ThanosQuerier{
ObjectMeta: metav1.ObjectMeta{