CSPL-3551 Enhancing Ingestor inputs

Author: kasiakoziol

api/v4/ingestorcluster_types.go

@@ -20,8 +20,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/types"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -47,6 +45,9 @@ type IngestorClusterSpec struct {
 	// Push Bus spec
 	PushBus PushBusSpec `json:"pushBus"`
 
+	// Pipeline configuration
+	PipelineConfig PipelineConfigSpec `json:"pipelineConfig"`
+
 	// Service account name
 	ServiceAccountName string `json:"serviceAccountName"`
 }
@@ -57,8 +58,6 @@ type PushBusSpec struct {
 	Type string `json:"type"`
 
 	SQS SQSSpec `json:"sqs"`
-
-	PipelineConfig PipelineConfigSpec `json:"pipelineConfig"`
 }
 
 type SQSSpec struct {
@@ -67,6 +66,18 @@ type SQSSpec struct {
 	AuthRegion string `json:"authRegion"`
 
 	Endpoint string `json:"endpoint"`
+
+	LargeMessageStoreEndpoint string `json:"largeMessageStoreEndpoint"`
+
+	LargeMessageStorePath string `json:"largeMessageStorePath"`
+
+	DeadLetterQueueName string `json:"deadLetterQueueName"`
+
+	MaxRetriesPerPart int `json:"maxRetriesPerPart"`
+
+	RetryPolicy string `json:"retryPolicy"`
+
+	SendInterval string `json:"sendInterval"`
 }
 
 type PipelineConfigSpec struct {
@@ -134,195 +145,12 @@ type IngestorCluster struct {
 	Status IngestorClusterStatus `json:"status,omitempty,omitzero"`
 }
 
-// DeepCopyObject implements client.Object.
-func (ic *IngestorCluster) DeepCopyObject() runtime.Object {
-	panic("unimplemented")
-}
-
-// GetAnnotations implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetAnnotations of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetAnnotations() map[string]string {
-	panic("unimplemented")
-}
-
-// GetCreationTimestamp implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetCreationTimestamp of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetCreationTimestamp() metav1.Time {
-	panic("unimplemented")
-}
-
-// GetDeletionGracePeriodSeconds implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetDeletionGracePeriodSeconds of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetDeletionGracePeriodSeconds() *int64 {
-	panic("unimplemented")
-}
-
-// GetDeletionTimestamp implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetDeletionTimestamp of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetDeletionTimestamp() *metav1.Time {
-	panic("unimplemented")
-}
-
-// GetFinalizers implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetFinalizers of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetFinalizers() []string {
-	panic("unimplemented")
-}
-
-// GetGenerateName implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetGenerateName of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetGenerateName() string {
-	panic("unimplemented")
-}
-
-// GetGeneration implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetGeneration of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetGeneration() int64 {
-	panic("unimplemented")
-}
-
-// GetLabels implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetLabels of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetLabels() map[string]string {
-	panic("unimplemented")
-}
-
-// GetManagedFields implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetManagedFields of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetManagedFields() []metav1.ManagedFieldsEntry {
-	panic("unimplemented")
-}
-
-// GetName implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetName of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetName() string {
-	panic("unimplemented")
-}
-
-// GetNamespace implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetNamespace of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetNamespace() string {
-	panic("unimplemented")
-}
-
-// GetObjectKind implements client.Object.
-// Subtle: this method shadows the method (TypeMeta).GetObjectKind of IngestorCluster.TypeMeta.
-func (ic *IngestorCluster) GetObjectKind() schema.ObjectKind {
-	panic("unimplemented")
-}
-
-// GetOwnerReferences implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetOwnerReferences of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetOwnerReferences() []metav1.OwnerReference {
-	panic("unimplemented")
-}
-
-// GetResourceVersion implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetResourceVersion of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetResourceVersion() string {
-	panic("unimplemented")
-}
-
-// GetSelfLink implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetSelfLink of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetSelfLink() string {
-	panic("unimplemented")
-}
-
-// GetUID implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).GetUID of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) GetUID() types.UID {
-	panic("unimplemented")
-}
-
-// SetAnnotations implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetAnnotations of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetAnnotations(annotations map[string]string) {
-	panic("unimplemented")
-}
-
-// SetCreationTimestamp implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetCreationTimestamp of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetCreationTimestamp(timestamp metav1.Time) {
-	panic("unimplemented")
-}
-
-// SetDeletionGracePeriodSeconds implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetDeletionGracePeriodSeconds of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetDeletionGracePeriodSeconds(*int64) {
-	panic("unimplemented")
-}
-
-// SetDeletionTimestamp implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetDeletionTimestamp of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetDeletionTimestamp(timestamp *metav1.Time) {
-	panic("unimplemented")
-}
-
-// SetFinalizers implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetFinalizers of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetFinalizers(finalizers []string) {
-	panic("unimplemented")
-}
-
-// SetGenerateName implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetGenerateName of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetGenerateName(name string) {
-	panic("unimplemented")
-}
-
-// SetGeneration implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetGeneration of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetGeneration(generation int64) {
-	panic("unimplemented")
-}
-
-// SetLabels implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetLabels of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetLabels(labels map[string]string) {
-	panic("unimplemented")
-}
-
-// SetManagedFields implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetManagedFields of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetManagedFields(managedFields []metav1.ManagedFieldsEntry) {
-	panic("unimplemented")
-}
-
-// SetName implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetName of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetName(name string) {
-	panic("unimplemented")
-}
-
-// SetNamespace implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetNamespace of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetNamespace(namespace string) {
-	panic("unimplemented")
-}
-
-// SetOwnerReferences implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetOwnerReferences of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetOwnerReferences([]metav1.OwnerReference) {
-	panic("unimplemented")
-}
-
-// SetResourceVersion implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetResourceVersion of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetResourceVersion(version string) {
-	panic("unimplemented")
-}
-
-// SetSelfLink implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetSelfLink of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetSelfLink(selfLink string) {
-	panic("unimplemented")
-}
-
-// SetUID implements client.Object.
-// Subtle: this method shadows the method (ObjectMeta).SetUID of IngestorCluster.ObjectMeta.
-func (ic *IngestorCluster) SetUID(uid types.UID) {
-	panic("unimplemented")
+// DeepCopyObject implements runtime.Object
+func (in *IngestorCluster) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
 }
 
 // +kubebuilder:object:root=true

api/v4/zz_generated.deepcopy.go

@@ -61,7 +61,6 @@ spec:
             type: object
           spec:
             description: IngestorClusterSpec defines the spec of Ingestor Cluster
-              pods
             properties:
               Mock:
                 description: Mock to differentiate between UTs and actual reconcile
@@ -1581,32 +1580,45 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              pipelineConfig:
+                description: Pipeline configuration
+                properties:
+                  indexerPipe:
+                    type: boolean
+                  remoteQueueOutput:
+                    type: boolean
+                  remoteQueueRuleset:
+                    type: boolean
+                  remoteQueueTyping:
+                    type: boolean
+                  ruleSet:
+                    type: boolean
+                  typing:
+                    type: boolean
+                type: object
               pushBus:
                 description: Push Bus spec
                 properties:
-                  pipelineConfig:
-                    properties:
-                      indexerPipe:
-                        type: boolean
-                      remoteQueueOutput:
-                        type: boolean
-                      remoteQueueRuleset:
-                        type: boolean
-                      remoteQueueTyping:
-                        type: boolean
-                      ruleSet:
-                        type: boolean
-                      typing:
-                        type: boolean
-                    type: object
                   sqs:
                     properties:
                       authRegion:
                         type: string
+                      deadLetterQueueName:
+                        type: string
                       endpoint:
                         type: string
+                      largeMessageStoreEndpoint:
+                        type: string
+                      largeMessageStorePath:
+                        type: string
+                      maxRetriesPerPart:
+                        type: integer
                       queueName:
                         type: string
+                      retryPolicy:
+                        type: string
+                      sendInterval:
+                        type: string
                     type: object
                   type:
                     type: string
@@ -1644,7 +1656,7 @@ spec:
                     type: integer
                 type: object
               replicas:
-                description: Number of ingestion pods
+                description: Number of ingestor pods
                 format: int32
                 type: integer
               resources:
@@ -4253,7 +4265,7 @@ spec:
             type: object
           status:
             description: IngestorClusterStatus defines the observed state of Ingestor
-              Cluster pods
+              Cluster
             properties:
               appContext:
                 description: App Framework context
@@ -4540,7 +4552,7 @@ spec:
                 description: Auxillary message describing CR status
                 type: string
               phase:
-                description: Phase of the ingestion pods
+                description: Phase of the ingestor pods
                 enum:
                 - Pending
                 - Ready
@@ -4551,11 +4563,11 @@ spec:
                 - Error
                 type: string
               readyReplicas:
-                description: Number of ready ingestion pods
+                description: Number of ready ingestor pods
                 format: int32
                 type: integer
               replicas:
-                description: Number of desired ingestion pods
+                description: Number of desired ingestor pods
                 format: int32
                 type: integer
               resourceRevMap:

config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml

@@ -153,6 +153,8 @@ func getTelAppNameExtension(crKind string) (string, error) {
 		return "cmaster", nil
 	case "ClusterManager":
 		return "cmanager", nil
+	case "IngestorCluster":
+		return "ingestor", nil
 	default:
 		return "", errors.New("Invalid CR kind for telemetry app")
 	}