Skip to content

Commit 635415e

Browse files
mateusz834mateusz834
committed
Deflake test_collection.py
Current tests perform read operations twice and assert based on previous results. This approach is inherently flaky because some background operations in Splunk can modify data between reads. This change, wraps every test case in assertEventuallyTrue.
1 parent 1aa5b20 commit 635415e

File tree

1 file changed

+161
-111
lines changed

1 file changed

+161
-111
lines changed

‎tests/test_collection.py‎

Lines changed: 161 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -83,109 +83,149 @@ def test_metadata(self):
8383

8484
def test_list(self):
8585
for coll_name in collections:
86-
coll = getattr(self.service, coll_name)
87-
expected = [ent.name for ent in coll.list(count=10, sort_mode="auto")]
88-
if len(expected) == 0:
89-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
90-
found = [ent.name for ent in coll.list()][:10]
91-
self.assertEqual(
92-
expected,
93-
found,
94-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
95-
)
86+
87+
def test():
88+
coll = getattr(self.service, coll_name)
89+
expected = [ent.name for ent in coll.list(count=10, sort_mode="auto")]
90+
found = [ent.name for ent in coll.list()][:10]
91+
if expected != found:
92+
logging.warning(
93+
f"on {coll_name} (expected: {expected}, found: {found})",
94+
)
95+
return False
96+
97+
return True
98+
99+
self.assertEventuallyTrue(test)
96100

97101
def test_list_with_count(self):
98-
N = 5
99102
for coll_name in collections:
100-
coll = getattr(self.service, coll_name)
101-
expected = [ent.name for ent in coll.list(count=N + 5)][:N]
102-
N = len(expected) # in case there are <N elements
103-
found = [ent.name for ent in coll.list(count=N)]
104-
self.assertEqual(
105-
expected,
106-
found,
107-
msg=f"on {coll_name} (expected {expected}, found {found})",
108-
)
103+
104+
def test():
105+
N = 5
106+
coll = getattr(self.service, coll_name)
107+
expected = [ent.name for ent in coll.list(count=N + 5)][:N]
108+
N = len(expected) # in case there are <N elements
109+
found = [ent.name for ent in coll.list(count=N)]
110+
if expected != found:
111+
logging.warning(
112+
f"on {coll_name} (expected: {expected}, found: {found})",
113+
)
114+
return False
115+
116+
return True
117+
118+
self.assertEventuallyTrue(test)
109119

110120
def test_list_with_offset(self):
111121
import random
112122

113123
for offset in [random.randint(3, 50) for x in range(5)]:
114124
for coll_name in collections:
115-
coll = getattr(self.service, coll_name)
116-
expected = [ent.name for ent in coll.list(count=offset + 10)][offset:]
117-
found = [ent.name for ent in coll.list(offset=offset, count=10)]
118-
self.assertEqual(
119-
expected,
120-
found,
121-
msg=f"on {coll_name} (expected {expected}, found {found})",
122-
)
125+
126+
def test():
127+
coll = getattr(self.service, coll_name)
128+
expected = [ent.name for ent in coll.list(count=offset + 10)][
129+
offset:
130+
]
131+
found = [ent.name for ent in coll.list(offset=offset, count=10)]
132+
if expected != found:
133+
logging.warning(
134+
f"on {coll_name} (expected: {expected}, found: {found})",
135+
)
136+
return False
137+
138+
return True
139+
140+
self.assertEventuallyTrue(test)
123141

124142
def test_list_with_search(self):
125143
for coll_name in collections:
126-
coll = getattr(self.service, coll_name)
127-
expected = [ent.name for ent in coll.list()]
128-
if len(expected) == 0:
129-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
130-
# TODO: DVPL-5868 - This should use a real search instead of *. Otherwise the test passes trivially.
131-
found = [ent.name for ent in coll.list(search="*")]
132-
self.assertEqual(
133-
expected,
134-
found,
135-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
136-
)
144+
145+
def test():
146+
coll = getattr(self.service, coll_name)
147+
expected = [ent.name for ent in coll.list()]
148+
# TODO: DVPL-5868 - This should use a real search instead of *. Otherwise the test passes trivially.
149+
found = [ent.name for ent in coll.list(search="*")]
150+
if expected != found:
151+
logging.warning(
152+
f"on {coll_name} (expected: {expected}, found: {found})",
153+
)
154+
return False
155+
156+
return True
157+
158+
self.assertEventuallyTrue(test)
137159

138160
def test_list_with_sort_dir(self):
139161
for coll_name in collections:
140-
coll = getattr(self.service, coll_name)
141-
expected_kwargs = {"sort_dir": "desc"}
142-
found_kwargs = {"sort_dir": "asc"}
143-
if coll_name == "jobs":
144-
expected_kwargs["sort_key"] = "sid"
145-
found_kwargs["sort_key"] = "sid"
146-
expected = list(
147-
reversed([ent.name for ent in coll.list(**expected_kwargs)])
148-
)
149-
if len(expected) == 0:
150-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
151-
found = [ent.name for ent in coll.list(**found_kwargs)]
152162

153-
self.assertEqual(
154-
sorted(expected),
155-
sorted(found),
156-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
157-
)
163+
def test():
164+
coll = getattr(self.service, coll_name)
165+
expected_kwargs = {"sort_dir": "desc"}
166+
found_kwargs = {"sort_dir": "asc"}
167+
if coll_name == "jobs":
168+
expected_kwargs["sort_key"] = "sid"
169+
found_kwargs["sort_key"] = "sid"
170+
expected = list(
171+
reversed([ent.name for ent in coll.list(**expected_kwargs)])
172+
)
173+
if len(expected) == 0:
174+
logging.debug(
175+
f"No entities in collection {coll_name}; skipping test."
176+
)
177+
found = [ent.name for ent in coll.list(**found_kwargs)]
178+
179+
if expected != found:
180+
logging.warning(
181+
f"on {coll_name} (expected: {expected}, found: {found})",
182+
)
183+
return False
184+
185+
return True
186+
187+
self.assertEventuallyTrue(test)
158188

159189
def test_list_with_sort_mode_auto(self):
160190
# The jobs collection requires special handling. The sort_dir kwarg is
161191
# needed because the default sorting direction for jobs is "desc", not
162192
# "asc". The sort_key kwarg is required because there is no default
163193
# sort_key for jobs in Splunk 6.
164194
for coll_name in collections:
165-
coll = getattr(self.service, coll_name)
166-
if coll_name == "jobs":
167-
expected = [
168-
ent.name
169-
for ent in coll.list(
170-
sort_mode="auto", sort_dir="asc", sort_key="sid"
195+
196+
def test():
197+
coll = getattr(self.service, coll_name)
198+
if coll_name == "jobs":
199+
expected = [
200+
ent.name
201+
for ent in coll.list(
202+
sort_mode="auto", sort_dir="asc", sort_key="sid"
203+
)
204+
]
205+
else:
206+
expected = [ent.name for ent in coll.list(sort_mode="auto")]
207+
208+
if len(expected) == 0:
209+
logging.debug(
210+
f"No entities in collection {coll_name}; skipping test."
171211
)
172-
]
173-
else:
174-
expected = [ent.name for ent in coll.list(sort_mode="auto")]
175212

176-
if len(expected) == 0:
177-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
213+
if coll_name == "jobs":
214+
found = [
215+
ent.name for ent in coll.list(sort_dir="asc", sort_key="sid")
216+
]
217+
else:
218+
found = [ent.name for ent in coll.list()]
178219

179-
if coll_name == "jobs":
180-
found = [ent.name for ent in coll.list(sort_dir="asc", sort_key="sid")]
181-
else:
182-
found = [ent.name for ent in coll.list()]
220+
if expected != found:
221+
logging.warning(
222+
f"on {coll_name} (expected: {expected}, found: {found})",
223+
)
224+
return False
183225

184-
self.assertEqual(
185-
expected,
186-
found,
187-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
188-
)
226+
return True
227+
228+
self.assertEventuallyTrue(test)
189229

190230
def test_list_with_sort_mode_alpha_case(self):
191231
for coll_name in collections:
@@ -228,44 +268,54 @@ def test_list_with_sort_mode_alpha(self):
228268

229269
def test_iteration(self):
230270
for coll_name in collections:
231-
coll = getattr(self.service, coll_name)
232-
expected = [ent.name for ent in coll.list(count=10)]
233-
if len(expected) == 0:
234-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
235-
total = len(expected)
236-
found = []
237-
for ent in coll.iter(pagesize=max(int(total / 5.0), 1), count=10):
238-
found.append(ent.name)
239-
self.assertEqual(
240-
expected,
241-
found,
242-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
243-
)
271+
272+
def test():
273+
coll = getattr(self.service, coll_name)
274+
expected = [ent.name for ent in coll.list(count=10)]
275+
total = len(expected)
276+
found = []
277+
for ent in coll.iter(pagesize=max(int(total / 5.0), 1), count=10):
278+
found.append(ent.name)
279+
if expected != found:
280+
logging.warning(
281+
f"on {coll_name} (expected: {expected}, found: {found})",
282+
)
283+
return False
284+
285+
return True
286+
287+
self.assertEventuallyTrue(test)
244288

245289
def test_paging(self):
246290
for coll_name in collections:
247-
coll = getattr(self.service, coll_name)
248-
expected = [ent.name for ent in coll.list(count=30)]
249-
if len(expected) == 0:
250-
logging.debug(f"No entities in collection {coll_name}; skipping test.")
251-
total = len(expected)
252-
page_size = max(int(total / 5.0), 1)
253-
found = []
254-
offset = 0
255-
while offset < total:
256-
page = coll.list(offset=offset, count=page_size)
257-
count = len(page)
258-
offset += count
259-
self.assertTrue(
260-
count == page_size or offset == total, msg=f"on {coll_name}"
261-
)
262-
found.extend([ent.name for ent in page])
263-
logging.debug("Iterate: offset=%d/%d", offset, total)
264-
self.assertEqual(
265-
expected,
266-
found,
267-
msg=f"on {coll_name} (expected: {expected}, found: {found})",
268-
)
291+
292+
def test():
293+
coll = getattr(self.service, coll_name)
294+
expected = [ent.name for ent in coll.list(count=30)]
295+
total = len(expected)
296+
page_size = max(int(total / 5.0), 1)
297+
found = []
298+
offset = 0
299+
while offset < total:
300+
page = coll.list(offset=offset, count=page_size)
301+
count = len(page)
302+
offset += count
303+
if count != page_size and offset != total:
304+
logging.warning(
305+
f"on {coll_name} (count = {count}, page_size = {page_size}, offset = {offset}, total = {total})",
306+
)
307+
return False
308+
found.extend([ent.name for ent in page])
309+
logging.debug("Iterate: offset=%d/%d", offset, total)
310+
if expected != found:
311+
logging.warning(
312+
f"on {coll_name} (expected: {expected}, found: {found})",
313+
)
314+
return False
315+
316+
return True
317+
318+
self.assertEventuallyTrue(test)
269319

270320
def test_getitem_with_nonsense(self):
271321
for coll_name in collections:

0 commit comments

Comments
 (0)