Skip to content

Commit a12f605

Browse files
mateusz834mateusz834
committed
Deflake test_collection.py
Current tests perform read operations twice and assert based on previous results. This approach is inherently flaky because some background operations in Splunk can modify data between reads. This change, wraps every test case in assertEventuallyTrue.
1 parent a84c951 commit a12f605

File tree

1 file changed

+97
-62
lines changed

1 file changed

+97
-62
lines changed

‎tests/test_collection.py‎

Lines changed: 97 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -82,61 +82,77 @@ def test_metadata(self):
8282
)
8383

8484
def test_list(self):
85-
for coll_name in collections:
85+
def test(coll_name):
8686
coll = getattr(self.service, coll_name)
8787
expected = [ent.name for ent in coll.list(count=10, sort_mode="auto")]
88-
if len(expected) == 0:
89-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
9088
found = [ent.name for ent in coll.list()][:10]
91-
self.assertEqual(
92-
expected,
93-
found,
94-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
95-
)
89+
if expected != found:
90+
logging.warning(
91+
f"on {coll_name} (expected: {expected}, found: {found})",
92+
)
93+
return False
94+
95+
return True
9696

97-
def test_list_with_count(self):
98-
N = 5
9997
for coll_name in collections:
98+
self.assertEventuallyTrue(lambda: test(coll_name))
99+
100+
def test_list_with_count(self):
101+
def test(coll_name):
102+
N = 5
100103
coll = getattr(self.service, coll_name)
101104
expected = [ent.name for ent in coll.list(count=N + 5)][:N]
102105
N = len(expected) # in case there are <N elements
103106
found = [ent.name for ent in coll.list(count=N)]
104-
self.assertEqual(
105-
expected,
106-
found,
107-
msg=f"on {coll_name} (expected {expected}, found {found})",
108-
)
107+
if expected != found:
108+
logging.warning(
109+
f"on {coll_name} (expected: {expected}, found: {found})",
110+
)
111+
return False
112+
113+
return True
114+
115+
for coll_name in collections:
116+
self.assertEventuallyTrue(lambda: test(coll_name))
109117

110118
def test_list_with_offset(self):
119+
def test(coll_name, offset):
120+
coll = getattr(self.service, coll_name)
121+
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
122+
found = [ent.name for ent in coll.list(offset=offset, count=10)]
123+
if expected != found:
124+
logging.warning(
125+
f"on {coll_name} (expected: {expected}, found: {found})",
126+
)
127+
return False
128+
129+
return True
130+
111131
import random
112132

113133
for offset in [random.randint(3, 50) for x in range(5)]:
114134
for coll_name in collections:
115-
coll = getattr(self.service, coll_name)
116-
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
117-
found = [ent.name for ent in coll.list(offset=offset, count=10)]
118-
self.assertEqual(
119-
expected,
120-
found,
121-
msg=f"on {coll_name} (expected {expected}, found {found})",
122-
)
135+
self.assertEventuallyTrue(lambda: test(coll_name, offset))
123136

124137
def test_list_with_search(self):
125-
for coll_name in collections:
138+
def test(coll_name):
126139
coll = getattr(self.service, coll_name)
127140
expected = [ent.name for ent in coll.list()]
128-
if len(expected) == 0:
129-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
130141
# TODO: DVPL-5868 - This should use a real search instead of *. Otherwise the test passes trivially.
131142
found = [ent.name for ent in coll.list(search="*")]
132-
self.assertEqual(
133-
expected,
134-
found,
135-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
136-
)
143+
if expected != found:
144+
logging.warning(
145+
f"on {coll_name} (expected: {expected}, found: {found})",
146+
)
147+
return False
148+
149+
return True
137150

138-
def test_list_with_sort_dir(self):
139151
for coll_name in collections:
152+
self.assertEventuallyTrue(lambda: test(coll_name))
153+
154+
def test_list_with_sort_dir(self):
155+
def test(coll_name):
140156
coll = getattr(self.service, coll_name)
141157
expected_kwargs = {"sort_dir": "desc"}
142158
found_kwargs = {"sort_dir": "asc"}
@@ -150,18 +166,24 @@ def test_list_with_sort_dir(self):
150166
logging.debug(f"No entities in collection {coll_name}; skipping test.")
151167
found = [ent.name for ent in coll.list(**found_kwargs)]
152168

153-
self.assertEqual(
154-
sorted(expected),
155-
sorted(found),
156-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
157-
)
169+
if expected != found:
170+
logging.warning(
171+
f"on {coll_name} (expected: {expected}, found: {found})",
172+
)
173+
return False
174+
175+
return True
176+
177+
for coll_name in collections:
178+
self.assertEventuallyTrue(lambda: test(coll_name))
158179

159180
def test_list_with_sort_mode_auto(self):
160181
# The jobs collection requires special handling. The sort_dir kwarg is
161182
# needed because the default sorting direction for jobs is "desc", not
162183
# "asc". The sort_key kwarg is required because there is no default
163184
# sort_key for jobs in Splunk 6.
164-
for coll_name in collections:
185+
186+
def test(coll_name):
165187
coll = getattr(self.service, coll_name)
166188
if coll_name == "jobs":
167189
expected = [
@@ -181,11 +203,16 @@ def test_list_with_sort_mode_auto(self):
181203
else:
182204
found = [ent.name for ent in coll.list()]
183205

184-
self.assertEqual(
185-
expected,
186-
found,
187-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
188-
)
206+
if expected != found:
207+
logging.warning(
208+
f"on {coll_name} (expected: {expected}, found: {found})",
209+
)
210+
return False
211+
212+
return True
213+
214+
for coll_name in collections:
215+
self.assertEventuallyTrue(lambda: test(coll_name))
189216

190217
def test_list_with_sort_mode_alpha_case(self):
191218
for coll_name in collections:
@@ -227,27 +254,28 @@ def test_list_with_sort_mode_alpha(self):
227254
)
228255

229256
def test_iteration(self):
230-
for coll_name in collections:
257+
def test(coll_name):
231258
coll = getattr(self.service, coll_name)
232259
expected = [ent.name for ent in coll.list(count=10)]
233-
if len(expected) == 0:
234-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
235260
total = len(expected)
236261
found = []
237262
for ent in coll.iter(pagesize=max(int(total / 5.0), 1), count=10):
238263
found.append(ent.name)
239-
self.assertEqual(
240-
expected,
241-
found,
242-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
243-
)
264+
if expected != found:
265+
logging.warning(
266+
f"on {coll_name} (expected: {expected}, found: {found})",
267+
)
268+
return False
269+
270+
return True
244271

245-
def test_paging(self):
246272
for coll_name in collections:
273+
self.assertEventuallyTrue(lambda: test(coll_name))
274+
275+
def test_paging(self):
276+
def test(coll_name):
247277
coll = getattr(self.service, coll_name)
248278
expected = [ent.name for ent in coll.list(count=30)]
249-
if len(expected) == 0:
250-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
251279
total = len(expected)
252280
page_size = max(int(total / 5.0), 1)
253281
found = []
@@ -256,16 +284,23 @@ def test_paging(self):
256284
page = coll.list(offset=offset, count=page_size)
257285
count = len(page)
258286
offset += count
259-
self.assertTrue(
260-
count == page_size or offset == total, msg=f"on {coll_name}"
261-
)
287+
if count != page_size and offset != total:
288+
logging.warning(
289+
f"on {coll_name} (count = {count}, page_size = {page_size}, offset = {offset}, total = {total})",
290+
)
291+
return False
262292
found.extend([ent.name for ent in page])
263293
logging.debug("Iterate: offset=%d/%d", offset, total)
264-
self.assertEqual(
265-
expected,
266-
found,
267-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
268-
)
294+
if expected != found:
295+
logging.warning(
296+
f"on {coll_name} (expected: {expected}, found: {found})",
297+
)
298+
return False
299+
300+
return True
301+
302+
for coll_name in collections:
303+
self.assertEventuallyTrue(lambda: test(coll_name))
269304

270305
def test_getitem_with_nonsense(self):
271306
for coll_name in collections:

0 commit comments

Comments
 (0)