add retries on backend responses and tests to address flakiness in CI(#3266)

Author: vepatel

tests/suite/test_ac_policies.py

@@ -4,6 +4,7 @@
 from suite.utils.custom_resources_utils import read_custom_resource
 from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy
 from suite.utils.resources_utils import (
+    ensure_response_from_backend,
     get_last_reload_time,
     get_test_file_name,
     replace_configmap_from_yaml,
@@ -108,6 +109,9 @@ def test_deny_policy(
         """
         Test if ip (10.0.0.1) block-listing is working: default(no policy) -> deny
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},
@@ -174,6 +178,9 @@ def test_allow_policy(
         """
         Test if ip (10.0.0.1) allow-listing is working: default(no policy) -> allow
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},
@@ -234,6 +241,9 @@ def test_override_policy(
         """
         Test if ip allow-listing overrides block-listing: default(no policy) -> deny and allow
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},
@@ -279,6 +289,9 @@ def test_invalid_policy(
         """
         Test if invalid policy is applied then response is 500
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},
@@ -333,6 +346,9 @@ def test_deleted_policy(
         """
         Test if valid policy is deleted then response is 500
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},
@@ -388,6 +404,9 @@ def test_route_override_spec(
         """
         Test allow policy specified under routes overrides block in spec
         """
+        ensure_response_from_backend(
+            virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, {"X-Real-IP": "10.0.0.1"}
+        )
         resp = requests.get(
             virtual_server_setup.backend_1_url,
             headers={"host": virtual_server_setup.vs_host, "X-Real-IP": "10.0.0.1"},

tests/suite/test_transport_server_external_name.py

@@ -99,8 +99,8 @@ def test_template_config(
         ts_externalname_setup,
     ):
         wait_before_test()
-
         nginx_file_path = f"/etc/nginx/nginx.conf"
+        nginx_conf = ""
         nginx_conf = get_file_contents(
             kube_apis.v1, nginx_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace
         )
@@ -109,16 +109,25 @@ def test_template_config(
         ts_file_path = (
             f"/etc/nginx/stream-conf.d/ts_{transport_server_setup.namespace}_{transport_server_setup.name}.conf"
         )
-        ts_conf = get_file_contents(
-            kube_apis.v1, ts_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace
-        )
+        ts_conf = ""
+        retry = 0
+        while f"{ts_externalname_setup.external_host}:5353" not in ts_conf and retry < 5:
+            wait_before_test()
+            ts_conf = get_file_contents(
+                kube_apis.v1,
+                ts_file_path,
+                ts_externalname_setup.ic_pod_name,
+                ingress_controller_prerequisites.namespace,
+            )
+            retry = +1
 
         assert resolver_count == 2  # one for http and other for stream context
         assert (
             f"server {ts_externalname_setup.external_host}:5353 max_fails=1 fail_timeout=10s max_conns=0 resolve;"
             in ts_conf
         )
 
+    @pytest.mark.flaky(max_runs=3)
     def test_event_warning(
         self,
         kube_apis,

tests/suite/test_v_s_route.py

@@ -311,7 +311,7 @@ def test_make_existing_vsr_invalid(
         patch_v_s_route_from_yaml(
             kube_apis.custom_objects, v_s_route_setup.route_s.name, route_yaml, v_s_route_setup.route_s.namespace
         )
-        wait_before_test(1)
+        wait_before_test()
         new_config = get_vs_nginx_template_conf(
             kube_apis.v1,
             v_s_route_setup.namespace,

tests/suite/test_v_s_route_canned_responses.py

@@ -109,6 +109,7 @@ def test_update(self, kube_apis, crd_ingress_controller, v_s_route_setup):
         assert_event_count_increased(vs_event_text, initial_count_vs, new_events_ns)
         assert_event_count_increased(vsr_event_text, initial_count_vsr, new_events_ns)
 
+    @pytest.mark.flaky(max_runs=3)
     def test_validation_flow(self, kube_apis, crd_ingress_controller, v_s_route_setup):
         invalid_fields = ["spec.subroutes[0].action.return.code", "spec.subroutes[0].action.return.body"]
         req_host = f"{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}"

tests/suite/test_v_s_route_grpc.py

@@ -134,6 +134,7 @@ def test_config_after_enable_tls(
         )
         assert "grpc_pass grpcs://" in config
 
+    @pytest.mark.flaky(max_runs=3)
     @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs"}], indirect=True)
     def test_validation_flow(
         self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, backend_setup, v_s_route_setup

tests/suite/test_v_s_route_redirects.py

@@ -78,6 +78,7 @@ def test_update(self, kube_apis, crd_ingress_controller, v_s_route_setup):
         assert_event_count_increased(vs_event_text, initial_count_vs, new_events_ns)
         assert_event_count_increased(vsr_event_text, initial_count_vsr, new_events_ns)
 
+    @pytest.mark.flaky(max_runs=3)
     def test_validation_flow(self, kube_apis, crd_ingress_controller, v_s_route_setup):
         req_host = f"{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}"
         req_url = f"http://{req_host}{v_s_route_setup.route_s.paths[0]}"

tests/suite/test_v_s_route_upstream_tls.py

@@ -123,6 +123,7 @@ def test_events_after_setup(
         assert_event(vsr_m_event_text, events_ns_m)
         assert_event(vs_event_text, events_ns_m)
 
+    @pytest.mark.flaky(max_runs=3)
     def test_validation_flow(
         self,
         kube_apis,

tests/suite/test_virtual_server_canned_responses.py

@@ -98,6 +98,7 @@ def test_update(self, kube_apis, crd_ingress_controller, virtual_server_setup):
         vs_events = get_events(kube_apis.v1, virtual_server_setup.namespace)
         assert_event_count_increased(vs_event_text, initial_count, vs_events)
 
+    @pytest.mark.flaky(max_runs=3)
     def test_validation_flow(self, kube_apis, crd_ingress_controller, virtual_server_setup):
         invalid_fields = ["spec.routes[0].action.return.code", "spec.routes[0].action.return.body"]
         text = f"{virtual_server_setup.namespace}/{virtual_server_setup.vs_name}"

tests/suite/test_virtual_server_error_pages.py

@@ -1,4 +1,6 @@
 import json
+from unittest import mock
+from unittest.mock import Mock
 
 import pytest
 import requests
@@ -29,9 +31,21 @@ def test_redirect_strategy(self, kube_apis, crd_ingress_controller, virtual_serv
         wait_and_assert_status_code(
             307, virtual_server_setup.backend_1_url, virtual_server_setup.vs_host, allow_redirects=False
         )
-        resp = requests.get(
-            virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host}, allow_redirects=False
-        )
+        retry = 0
+        while retry < 5:
+            wait_before_test()
+            try:
+                resp = requests.get(
+                    virtual_server_setup.backend_1_url,
+                    headers={"host": virtual_server_setup.vs_host},
+                    allow_redirects=False,
+                )
+                print(f"redirect to uri: {resp.next.url}")
+            except AttributeError as e:
+                print(f"Exception occured: {e}")
+                retry = +1
+                continue
+            break
         assert f"http://{virtual_server_setup.vs_host}/error.html" in resp.next.url
 
     def test_return_strategy(self, kube_apis, crd_ingress_controller, virtual_server_setup):

tests/suite/test_virtual_server_grpc.py

@@ -115,6 +115,7 @@ def test_config_after_setup(
         assert_grpc_entries_exist(config)
         assert_proxy_entries_do_not_exist(config)
 
+    @pytest.mark.flaky(max_runs=3)
     @pytest.mark.parametrize("backend_setup", [{"app_type": "grpc-vs"}], indirect=True)
     def test_validation_flow(
         self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, backend_setup, virtual_server_setup