Skip to content

Commit 3fde468

Browse files
mateusz834mateusz834
committed
Deflake test_collection.py
Current tests perform read operations twice and assert based on previous results. This approach is inherently flaky because some background operations in Splunk can modify data between reads. This change, wraps every test case in assertEventuallyTrue.
1 parent a84c951 commit 3fde468

File tree

1 file changed

+111
-73
lines changed

1 file changed

+111
-73
lines changed

‎tests/test_collection.py‎

Lines changed: 111 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -82,86 +82,111 @@ def test_metadata(self):
8282
)
8383

8484
def test_list(self):
85-
for coll_name in collections:
85+
def test(coll_name):
8686
coll = getattr(self.service, coll_name)
8787
expected = [ent.name for ent in coll.list(count=10, sort_mode="auto")]
88-
if len(expected) == 0:
89-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
9088
found = [ent.name for ent in coll.list()][:10]
91-
self.assertEqual(
92-
expected,
93-
found,
94-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
95-
)
89+
if expected != found:
90+
logging.warning(
91+
f"on {coll_name} (expected: {expected}, found: {found})",
92+
)
93+
return False
94+
95+
return True
9696

97-
def test_list_with_count(self):
98-
N = 5
9997
for coll_name in collections:
98+
self.assertEventuallyTrue(lambda: test(coll_name))
99+
100+
def test_list_with_count(self):
101+
def test(coll_name):
102+
N = 5
100103
coll = getattr(self.service, coll_name)
101104
expected = [ent.name for ent in coll.list(count=N + 5)][:N]
102105
N = len(expected) # in case there are <N elements
103106
found = [ent.name for ent in coll.list(count=N)]
104-
self.assertEqual(
105-
expected,
106-
found,
107-
msg=f"on {coll_name} (expected {expected}, found {found})",
108-
)
107+
if expected != found:
108+
logging.warning(
109+
f"on {coll_name} (expected: {expected}, found: {found})",
110+
)
111+
return False
112+
113+
return True
114+
115+
for coll_name in collections:
116+
self.assertEventuallyTrue(lambda: test(coll_name))
109117

110118
def test_list_with_offset(self):
119+
def test(coll_name, offset):
120+
coll = getattr(self.service, coll_name)
121+
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
122+
found = [ent.name for ent in coll.list(offset=offset, count=10)]
123+
if expected != found:
124+
logging.warning(
125+
f"on {coll_name} (expected: {expected}, found: {found})",
126+
)
127+
return False
128+
129+
return True
130+
111131
import random
112132

113133
for offset in [random.randint(3, 50) for x in range(5)]:
114134
for coll_name in collections:
115-
coll = getattr(self.service, coll_name)
116-
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
117-
found = [ent.name for ent in coll.list(offset=offset, count=10)]
118-
self.assertEqual(
119-
expected,
120-
found,
121-
msg=f"on {coll_name} (expected {expected}, found {found})",
122-
)
135+
self.assertEventuallyTrue(lambda: test(coll_name, offset))
123136

124137
def test_list_with_search(self):
125-
for coll_name in collections:
138+
def test(coll_name):
126139
coll = getattr(self.service, coll_name)
127140
expected = [ent.name for ent in coll.list()]
128-
if len(expected) == 0:
129-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
130141
# TODO: DVPL-5868 - This should use a real search instead of *. Otherwise the test passes trivially.
131142
found = [ent.name for ent in coll.list(search="*")]
132-
self.assertEqual(
133-
expected,
134-
found,
135-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
136-
)
143+
if expected != found:
144+
logging.warning(
145+
f"on {coll_name} (expected: {expected}, found: {found})",
146+
)
147+
return False
148+
149+
return True
150+
151+
for coll_name in collections:
152+
self.assertEventuallyTrue(lambda: test(coll_name))
137153

138154
def test_list_with_sort_dir(self):
139155
for coll_name in collections:
140-
coll = getattr(self.service, coll_name)
141-
expected_kwargs = {"sort_dir": "desc"}
142-
found_kwargs = {"sort_dir": "asc"}
143-
if coll_name == "jobs":
144-
expected_kwargs["sort_key"] = "sid"
145-
found_kwargs["sort_key"] = "sid"
146-
expected = list(
147-
reversed([ent.name for ent in coll.list(**expected_kwargs)])
148-
)
149-
if len(expected) == 0:
150-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
151-
found = [ent.name for ent in coll.list(**found_kwargs)]
152156

153-
self.assertEqual(
154-
sorted(expected),
155-
sorted(found),
156-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
157-
)
157+
def test(coll_name):
158+
coll = getattr(self.service, coll_name)
159+
expected_kwargs = {"sort_dir": "desc"}
160+
found_kwargs = {"sort_dir": "asc"}
161+
if coll_name == "jobs":
162+
expected_kwargs["sort_key"] = "sid"
163+
found_kwargs["sort_key"] = "sid"
164+
expected = list(
165+
reversed([ent.name for ent in coll.list(**expected_kwargs)])
166+
)
167+
if len(expected) == 0:
168+
logging.debug(
169+
f"No entities in collection {coll_name}; skipping test."
170+
)
171+
found = [ent.name for ent in coll.list(**found_kwargs)]
172+
173+
if expected != found:
174+
logging.warning(
175+
f"on {coll_name} (expected: {expected}, found: {found})",
176+
)
177+
return False
178+
179+
return True
180+
181+
self.assertEventuallyTrue(lambda: test(coll_name))
158182

159183
def test_list_with_sort_mode_auto(self):
160184
# The jobs collection requires special handling. The sort_dir kwarg is
161185
# needed because the default sorting direction for jobs is "desc", not
162186
# "asc". The sort_key kwarg is required because there is no default
163187
# sort_key for jobs in Splunk 6.
164-
for coll_name in collections:
188+
189+
def test(coll_name):
165190
coll = getattr(self.service, coll_name)
166191
if coll_name == "jobs":
167192
expected = [
@@ -181,11 +206,16 @@ def test_list_with_sort_mode_auto(self):
181206
else:
182207
found = [ent.name for ent in coll.list()]
183208

184-
self.assertEqual(
185-
expected,
186-
found,
187-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
188-
)
209+
if expected != found:
210+
logging.warning(
211+
f"on {coll_name} (expected: {expected}, found: {found})",
212+
)
213+
return False
214+
215+
return True
216+
217+
for coll_name in collections:
218+
self.assertEventuallyTrue(lambda: test(coll_name))
189219

190220
def test_list_with_sort_mode_alpha_case(self):
191221
for coll_name in collections:
@@ -227,27 +257,28 @@ def test_list_with_sort_mode_alpha(self):
227257
)
228258

229259
def test_iteration(self):
230-
for coll_name in collections:
260+
def test(coll_name):
231261
coll = getattr(self.service, coll_name)
232262
expected = [ent.name for ent in coll.list(count=10)]
233-
if len(expected) == 0:
234-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
235263
total = len(expected)
236264
found = []
237265
for ent in coll.iter(pagesize=max(int(total / 5.0), 1), count=10):
238266
found.append(ent.name)
239-
self.assertEqual(
240-
expected,
241-
found,
242-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
243-
)
267+
if expected != found:
268+
logging.warning(
269+
f"on {coll_name} (expected: {expected}, found: {found})",
270+
)
271+
return False
272+
273+
return True
244274

245-
def test_paging(self):
246275
for coll_name in collections:
276+
self.assertEventuallyTrue(lambda: test(coll_name))
277+
278+
def test_paging(self):
279+
def test(coll_name):
247280
coll = getattr(self.service, coll_name)
248281
expected = [ent.name for ent in coll.list(count=30)]
249-
if len(expected) == 0:
250-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
251282
total = len(expected)
252283
page_size = max(int(total / 5.0), 1)
253284
found = []
@@ -256,16 +287,23 @@ def test_paging(self):
256287
page = coll.list(offset=offset, count=page_size)
257288
count = len(page)
258289
offset += count
259-
self.assertTrue(
260-
count == page_size or offset == total, msg=f"on {coll_name}"
261-
)
290+
if count != page_size and offset != total:
291+
logging.warning(
292+
f"on {coll_name} (count = {count}, page_size = {page_size}, offset = {offset}, total = {total})",
293+
)
294+
return False
262295
found.extend([ent.name for ent in page])
263296
logging.debug("Iterate: offset=%d/%d", offset, total)
264-
self.assertEqual(
265-
expected,
266-
found,
267-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
268-
)
297+
if expected != found:
298+
logging.warning(
299+
f"on {coll_name} (expected: {expected}, found: {found})",
300+
)
301+
return False
302+
303+
return True
304+
305+
for coll_name in collections:
306+
self.assertEventuallyTrue(lambda: test(coll_name))
269307

270308
def test_getitem_with_nonsense(self):
271309
for coll_name in collections:

0 commit comments

Comments
 (0)