feat(slack): flag to disable slack integrations when fatal error (#54795)

Under flag` "organizations:slack-fatal-disable-on-broken"`
New method `is_integration_fatal_broken()` to only disable integrations
when "fatal" responses are received from slack: `"account_inactive"`

Author: chloeho7

src/sentry/features/__init__.py

@@ -264,7 +264,7 @@
 default_manager.add("organizations:ds-sliding-window", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:ds-sliding-window-org", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:ds-org-recalibration", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
-default_manager.add("organizations:slack-disable-on-broken", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
+default_manager.add("organizations:slack-fatal-disable-on-broken", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:sourcemaps-bundle-flat-file-indexing", OrganizationFeature, FeatureHandlerStrategy.REMOTE)
 default_manager.add("organizations:recap-server", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)
 default_manager.add("organizations:detailed-alert-logging", OrganizationFeature, FeatureHandlerStrategy.INTERNAL)

src/sentry/integrations/request_buffer.py

@@ -39,6 +39,25 @@ def record_fatal(self):
     def record_timeout(self):
         self._add("timeout")
 
+    def is_integration_fatal_broken(self):
+        """
+        Integration is broken if we have a fatal error
+        Temporary fix to release disabling slack integrations with fatal errors
+
+        """
+        broken_range_days_counts = self._get_broken_range_from_buffer()
+
+        days_fatal = []
+
+        for day_count in broken_range_days_counts:
+            if int(day_count.get("fatal_count", 0)) > 0:
+                days_fatal.append(day_count)
+
+        if len(days_fatal) > 0:
+            return True
+
+        return False
+
     def is_integration_broken(self):
         """
         Integration is broken if we have 7 consecutive days of errors and no successes OR have a fatal error OR have > 1000 timeouts in a day

src/sentry/shared_integrations/client/base.py

@@ -481,9 +481,9 @@ def disable_integration(self, buffer) -> None:
         )
 
         if (
-            features.has("organizations:slack-disable-on-broken", org)
+            features.has("organizations:slack-fatal-disable-on-broken", org)
             and rpc_integration.provider == "slack"
-        ):
+        ) and buffer.is_integration_fatal_broken():
             integration_service.update_integration(
                 integration_id=rpc_integration.id, status=ObjectStatus.DISABLED
             )

tests/sentry/integrations/slack/test_disable.py

@@ -48,7 +48,7 @@ def tearDown(self):
         self.resp.__exit__(None, None, None)
 
     @responses.activate
-    @with_feature("organizations:slack-disable-on-broken")
+    @with_feature("organizations:slack-fatal-disable-on-broken")
     def test_fatal_and_disable_integration(self):
         """
         fatal fast shut off with disable flag on, integration should be broken and disabled
@@ -73,7 +73,6 @@ def test_fatal_and_disable_integration(self):
         assert len(buffer._get_all_from_buffer()) == 0
 
     @responses.activate
-    @with_feature("organizations:disable-on-broken")
     def test_email(self):
         client = SlackClient(integration_id=self.integration.id)
         with self.tasks():
@@ -145,7 +144,7 @@ def test_error_integration(self):
         assert buffer.is_integration_broken() is False
 
     @responses.activate
-    @with_feature("organizations:slack-disable-on-broken")
+    @with_feature("organizations:slack-fatal-disable-on-broken")
     def test_slow_integration_is_not_broken_or_disabled(self):
         """
         slow test with disable flag on