feat: option to triggering reconciler on all events (#2894)

Signed-off-by: Attila Mészáros <[email protected]>

Author: csviri

docs/content/en/docs/documentation/reconciler.md

@@ -210,3 +210,52 @@ called, either by calling any of the `PrimeUpdateAndCacheUtils` methods again or
 updated via `PrimaryUpdateAndCacheUtils`.
 
 See related integration test [here](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/statuscache).
+
+### Trigger reconciliation for all events
+
+TLDR; We provide an execution mode where `reconcile` method is called on every event from event source.
+
+The framework optimizes execution for generic use cases, which, in almost all cases, fall into two categories:
+
+1. The controller does not use finalizers; thus when the primary resource is deleted, all the managed secondary
+   resources are cleaned up using the Kubernetes garbage collection mechanism, a.k.a., using owner references. This
+   mechanism, however, only works when all secondary resources are Kubernetes resources in the same namespace as the
+   primary resource.
+2. The controller uses finalizers (the controller implements the `Cleaner` interface), when explicit cleanup logic is
+   required, typically for external resources and when secondary resources are in different namespace than the primary
+   resources (owner references cannot be used in this case).
+
+Note that neither of those cases trigger the `reconcile` method of the controller on the `Delete` event of the primary
+resource. When a finalizer is used, the SDK calls the `cleanup` method of the `Cleaner` implementation when the resource
+is marked for deletion and the finalizer specified by the controller is present on the primary resource. When there is
+no finalizer, there is no need to call the `reconcile` method on a `Delete` event since all the cleanup will be done by
+the garbage collector. This avoids reconciliation cycles.
+
+However, there are cases when controllers do not strictly follow those patterns, typically when:
+
+- Only some of the primary resources use finalizers, e.g., for some of the primary resources you need
+  to create an external resource for others not.
+- You maintain some additional in memory caches (so not all the caches are encapsulated by an `EventSource`)
+  and you don't want to use finalizers. For those cases, you typically want to clean up your caches when the primary
+  resource is deleted.
+
+For such use cases you can set [`triggerReconcilerOnAllEvent`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java#L81)
+to `true`, as a result, the `reconcile` method will be triggered on ALL events (so also `Delete` events), making it
+possible to support the above use cases.
+
+In this mode:
+
+- even if the primary resource is already deleted from the Informer's cache, we will still pass the last known state
+  as the parameter for the reconciler. You can check if the resource is deleted using
+  `Context.isPrimaryResourceDeleted()`.
+- The retry, rate limiting, re-schedule, filters mechanisms work normally. The internal caches related to the resource
+  are cleaned up only when there is a successful reconciliation after a `Delete` event was received for the primary
+  resource
+  and reconciliation is not re-scheduled.
+- you cannot use the `Cleaner` interface. The framework assumes you will explicitly manage the finalizers. To
+  add finalizer you can use [
+  `PrimeUpdateAndCacheUtils`](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/PrimaryUpdateAndCacheUtils.java#L308).
+- you cannot use managed dependent resources since those manage the finalizers and other logic related to the normal
+  execution mode.
+
+See also [sample](https://github.com/operator-framework/java-operator-sdk/blob/main/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/triggerallevent/finalizerhandling) for selectively adding finalizers for resources;

operator-framework-core/pom.xml

@@ -79,6 +79,11 @@
       <artifactId>awaitility</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kube-api-test-client-inject</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/BaseConfigurationService.java

@@ -304,12 +304,15 @@ private <P extends HasMetadata> ResolvedControllerConfiguration<P> controllerCon
     final var dependentFieldManager =
         fieldManager.equals(CONTROLLER_NAME_AS_FIELD_MANAGER) ? name : fieldManager;
 
+    var triggerReconcilerOnAllEvent =
+        annotation != null && annotation.triggerReconcilerOnAllEvent();
+
     InformerConfiguration<P> informerConfig =
         InformerConfiguration.builder(resourceClass)
             .initFromAnnotation(annotation != null ? annotation.informer() : null, context)
             .buildForController();
 
-    return new ResolvedControllerConfiguration<P>(
+    return new ResolvedControllerConfiguration<>(
         name,
         generationAware,
         associatedReconcilerClass,
@@ -323,7 +326,8 @@ private <P extends HasMetadata> ResolvedControllerConfiguration<P> controllerCon
         null,
         dependentFieldManager,
         this,
-        informerConfig);
+        informerConfig,
+        triggerReconcilerOnAllEvent);
   }
 
   /**

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfiguration.java

@@ -92,4 +92,8 @@ default String fieldManager() {
   }
 
   <C> C getConfigurationFor(DependentResourceSpec<?, P, C> spec);
+
+  default boolean triggerReconcilerOnAllEvent() {
+    return false;
+  }
 }

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverrider.java

@@ -30,6 +30,7 @@ public class ControllerConfigurationOverrider<R extends HasMetadata> {
   private Duration reconciliationMaxInterval;
   private Map<DependentResourceSpec, Object> configurations;
   private final InformerConfiguration<R>.Builder config;
+  private boolean triggerReconcilerOnAllEvent;
 
   private ControllerConfigurationOverrider(ControllerConfiguration<R> original) {
     this.finalizer = original.getFinalizerName();
@@ -42,6 +43,7 @@ private ControllerConfigurationOverrider(ControllerConfiguration<R> original) {
     this.rateLimiter = original.getRateLimiter();
     this.name = original.getName();
     this.fieldManager = original.fieldManager();
+    this.triggerReconcilerOnAllEvent = original.triggerReconcilerOnAllEvent();
   }
 
   public ControllerConfigurationOverrider<R> withFinalizer(String finalizer) {
@@ -154,6 +156,12 @@ public ControllerConfigurationOverrider<R> withFieldManager(String dependentFiel
     return this;
   }
 
+  public ControllerConfigurationOverrider<R> withTriggerReconcilerOnAllEvent(
+      boolean triggerReconcilerOnAllEvent) {
+    this.triggerReconcilerOnAllEvent = triggerReconcilerOnAllEvent;
+    return this;
+  }
+
   /**
    * Sets a max page size limit when starting the informer. This will result in pagination while
    * populating the cache. This means that longer lists will take multiple requests to fetch. See
@@ -198,6 +206,7 @@ public ControllerConfiguration<R> build() {
         fieldManager,
         original.getConfigurationService(),
         config.buildForController(),
+        triggerReconcilerOnAllEvent,
         original.getWorkflowSpec().orElse(null));
   }
 

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ResolvedControllerConfiguration.java

@@ -29,6 +29,7 @@ public class ResolvedControllerConfiguration<P extends HasMetadata>
   private final Map<DependentResourceSpec, Object> configurations;
   private final ConfigurationService configurationService;
   private final String fieldManager;
+  private final boolean triggerReconcilerOnAllEvent;
   private WorkflowSpec workflowSpec;
 
   public ResolvedControllerConfiguration(ControllerConfiguration<P> other) {
@@ -44,6 +45,7 @@ public ResolvedControllerConfiguration(ControllerConfiguration<P> other) {
         other.fieldManager(),
         other.getConfigurationService(),
         other.getInformerConfig(),
+        other.triggerReconcilerOnAllEvent(),
         other.getWorkflowSpec().orElse(null));
   }
 
@@ -59,6 +61,7 @@ public ResolvedControllerConfiguration(
       String fieldManager,
       ConfigurationService configurationService,
       InformerConfiguration<P> informerConfig,
+      boolean triggerReconcilerOnAllEvent,
       WorkflowSpec workflowSpec) {
     this(
         name,
@@ -71,7 +74,8 @@ public ResolvedControllerConfiguration(
         configurations,
         fieldManager,
         configurationService,
-        informerConfig);
+        informerConfig,
+        triggerReconcilerOnAllEvent);
     setWorkflowSpec(workflowSpec);
   }
 
@@ -86,7 +90,8 @@ protected ResolvedControllerConfiguration(
       Map<DependentResourceSpec, Object> configurations,
       String fieldManager,
       ConfigurationService configurationService,
-      InformerConfiguration<P> informerConfig) {
+      InformerConfiguration<P> informerConfig,
+      boolean triggerReconcilerOnAllEvent) {
     this.informerConfig = informerConfig;
     this.configurationService = configurationService;
     this.name = ControllerConfiguration.ensureValidName(name, associatedReconcilerClassName);
@@ -99,6 +104,7 @@ protected ResolvedControllerConfiguration(
     this.finalizer =
         ControllerConfiguration.ensureValidFinalizerName(finalizer, getResourceTypeName());
     this.fieldManager = fieldManager;
+    this.triggerReconcilerOnAllEvent = triggerReconcilerOnAllEvent;
   }
 
   protected ResolvedControllerConfiguration(
@@ -117,7 +123,8 @@ protected ResolvedControllerConfiguration(
         null,
         null,
         configurationService,
-        InformerConfiguration.builder(resourceClass).buildForController());
+        InformerConfiguration.builder(resourceClass).buildForController(),
+        false);
   }
 
   @Override
@@ -207,4 +214,9 @@ public <C> C getConfigurationFor(DependentResourceSpec<?, P, C> spec) {
   public String fieldManager() {
     return fieldManager;
   }
+
+  @Override
+  public boolean triggerReconcilerOnAllEvent() {
+    return triggerReconcilerOnAllEvent;
+  }
 }

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/Context.java

@@ -72,4 +72,23 @@ default <R> Stream<R> getSecondaryResourcesAsStream(Class<R> expectedType) {
    * @return {@code true} is another reconciliation is already scheduled, {@code false} otherwise
    */
   boolean isNextReconciliationImminent();
+
+  /**
+   * To check if the primary resource is already deleted. This value can be true only if you turn on
+   * {@link
+   * io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration#triggerReconcilerOnAllEvent()}
+   *
+   * @return true Delete event received for primary resource
+   * @since 5.2.0
+   */
+  boolean isPrimaryResourceDeleted();
+
+  /**
+   * Check this only if {@link #isPrimaryResourceDeleted()} is true.
+   *
+   * @return true if the primary resource is deleted, but the last known state is only available
+   *     from the caches of the underlying Informer, not from Delete event.
+   * @since 5.2.0
+   */
+  boolean isPrimaryResourceFinalStateUnknown();
 }

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/ControllerConfiguration.java

@@ -77,4 +77,11 @@ MaxReconciliationInterval maxReconciliationInterval() default
    * @return the name used as field manager for SSA operations
    */
   String fieldManager() default CONTROLLER_NAME_AS_FIELD_MANAGER;
+
+  /**
+   * By settings to true, reconcile method will be triggered on every event, thus even for Delete
+   * event. You cannot use {@link Cleaner} or managed dependent resources in that case. See
+   * documentation for further details.
+   */
+  boolean triggerReconcilerOnAllEvent() default false;
 }

operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/reconciler/DefaultContext.java

@@ -24,12 +24,21 @@ public class DefaultContext<P extends HasMetadata> implements Context<P> {
   private final ControllerConfiguration<P> controllerConfiguration;
   private final DefaultManagedWorkflowAndDependentResourceContext<P>
       defaultManagedDependentResourceContext;
-
-  public DefaultContext(RetryInfo retryInfo, Controller<P> controller, P primaryResource) {
+  private final boolean primaryResourceDeleted;
+  private final boolean primaryResourceFinalStateUnknown;
+
+  public DefaultContext(
+      RetryInfo retryInfo,
+      Controller<P> controller,
+      P primaryResource,
+      boolean primaryResourceDeleted,
+      boolean primaryResourceFinalStateUnknown) {
     this.retryInfo = retryInfo;
     this.controller = controller;
     this.primaryResource = primaryResource;
     this.controllerConfiguration = controller.getConfiguration();
+    this.primaryResourceDeleted = primaryResourceDeleted;
+    this.primaryResourceFinalStateUnknown = primaryResourceFinalStateUnknown;
     this.defaultManagedDependentResourceContext =
         new DefaultManagedWorkflowAndDependentResourceContext<>(controller, primaryResource, this);
   }
@@ -119,6 +128,16 @@ public boolean isNextReconciliationImminent() {
         .isNextReconciliationImminent(ResourceID.fromResource(primaryResource));
   }
 
+  @Override
+  public boolean isPrimaryResourceDeleted() {
+    return primaryResourceDeleted;
+  }
+
+  @Override
+  public boolean isPrimaryResourceFinalStateUnknown() {
+    return primaryResourceFinalStateUnknown;
+  }
+
   public DefaultContext<P> setRetryInfo(RetryInfo retryInfo) {
     this.retryInfo = retryInfo;
     return this;