Skip to content

Commit 92714c8

Browse files
kuiwang02tmshort
kuiwang02
authored and
tmshort
committed
UPSTREAM: <carry>: migrate more cases from tests-private and enhance suites with filters
1 parent cfa5dd5 commit 92714c8

File tree

6 files changed

+1624
-20
lines changed

6 files changed

+1624
-20
lines changed

‎openshift/tests-extension/.openshift-tests-extension/openshift_payload_olmv1.json‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,219 @@
1616
},
1717
{
1818
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:83069-olmv1 static networkpolicy.",
19+
"labels": {
20+
"Extended": {},
21+
"NonHyperShiftHOST": {},
22+
"ReleaseGate": {}
23+
},
24+
"resources": {
25+
"isolation": {}
26+
},
27+
"source": "openshift:payload:olmv1",
28+
"lifecycle": "blocking",
29+
"environmentSelector": {
30+
"exclude": "topology==\"External\""
31+
}
32+
},
33+
{
34+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:68936-[Skipped:Disconnected]cluster extension can not be installed with insufficient permission sa for operand",
35+
"labels": {
36+
"Extended": {},
37+
"NonHyperShiftHOST": {}
38+
},
39+
"resources": {
40+
"isolation": {}
41+
},
42+
"source": "openshift:payload:olmv1",
43+
"lifecycle": "informing",
44+
"environmentSelector": {
45+
"exclude": "topology==\"External\""
46+
}
47+
},
48+
{
49+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:68937-[Skipped:Disconnected]cluster extension can not be installed with insufficient permission sa for operand rbac object",
50+
"labels": {
51+
"Extended": {},
52+
"NonHyperShiftHOST": {}
53+
},
54+
"resources": {
55+
"isolation": {}
56+
},
57+
"source": "openshift:payload:olmv1",
58+
"lifecycle": "informing",
59+
"environmentSelector": {
60+
"exclude": "topology==\"External\""
61+
}
62+
},
63+
{
64+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:75492-[Skipped:Disconnected]cluster extension can not be installed with wrong sa or insufficient permission sa",
65+
"labels": {
66+
"Extended": {},
67+
"LEVEL0": {},
68+
"NonHyperShiftHOST": {}
69+
},
70+
"resources": {
71+
"isolation": {}
72+
},
73+
"source": "openshift:payload:olmv1",
74+
"lifecycle": "informing",
75+
"environmentSelector": {
76+
"exclude": "topology==\"External\""
77+
}
78+
},
79+
{
80+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:75493-[Skipped:Disconnected]cluster extension can be installed with enough permission sa",
81+
"labels": {
82+
"Extended": {},
83+
"LEVEL0": {},
84+
"NonHyperShiftHOST": {}
85+
},
86+
"resources": {
87+
"isolation": {}
88+
},
89+
"source": "openshift:payload:olmv1",
90+
"lifecycle": "informing",
91+
"environmentSelector": {
92+
"exclude": "topology==\"External\""
93+
}
94+
},
95+
{
96+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:81538-[Skipped:Disconnected]preflight check on permission on allns mode",
97+
"labels": {
98+
"Extended": {},
99+
"NonHyperShiftHOST": {}
100+
},
101+
"resources": {
102+
"isolation": {}
103+
},
104+
"source": "openshift:payload:olmv1",
105+
"lifecycle": "informing",
106+
"environmentSelector": {
107+
"exclude": "topology==\"External\""
108+
}
109+
},
110+
{
111+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:81664-[Skipped:Disconnected]preflight check on permission on own ns mode",
112+
"labels": {
113+
"Extended": {},
114+
"NonHyperShiftHOST": {}
115+
},
116+
"resources": {
117+
"isolation": {}
118+
},
119+
"source": "openshift:payload:olmv1",
120+
"lifecycle": "informing",
121+
"environmentSelector": {
122+
"exclude": "topology==\"External\""
123+
}
124+
},
125+
{
126+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:81696-[Skipped:Disconnected]preflight check on permission on single ns mode",
127+
"labels": {
128+
"Extended": {},
129+
"NonHyperShiftHOST": {}
130+
},
131+
"resources": {
132+
"isolation": {}
133+
},
134+
"source": "openshift:payload:olmv1",
135+
"lifecycle": "informing",
136+
"environmentSelector": {
137+
"exclude": "topology==\"External\""
138+
}
139+
},
140+
{
141+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:74618-[Skipped:Disconnected]ClusterExtension supports simple registry vzero bundles only",
142+
"labels": {
143+
"Extended": {},
144+
"NonHyperShiftHOST": {}
145+
},
146+
"resources": {
147+
"isolation": {}
148+
},
149+
"source": "openshift:payload:olmv1",
150+
"lifecycle": "informing",
151+
"environmentSelector": {
152+
"exclude": "topology==\"External\""
153+
}
154+
},
155+
{
156+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:76843-[Skipped:Disconnected]support disc with icsp[Timeout:30m] [Serial][Disruptive][Slow]",
157+
"labels": {
158+
"Extended": {},
159+
"NonHyperShiftHOST": {}
160+
},
161+
"resources": {
162+
"isolation": {}
163+
},
164+
"source": "openshift:payload:olmv1",
165+
"lifecycle": "informing",
166+
"environmentSelector": {
167+
"exclude": "topology==\"External\""
168+
}
169+
},
170+
{
171+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:76844-[Skipped:Disconnected]support disc with itms and idms[Timeout:30m] [Serial][Disruptive][Slow]",
172+
"labels": {
173+
"Extended": {},
174+
"NonHyperShiftHOST": {}
175+
},
176+
"resources": {
177+
"isolation": {}
178+
},
179+
"source": "openshift:payload:olmv1",
180+
"lifecycle": "informing",
181+
"environmentSelector": {
182+
"exclude": "topology==\"External\""
183+
}
184+
},
185+
{
186+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:78193-[Skipped:Disconnected]Runtime validation of container images using sigstore signatures [Serial][Disruptive][Slow]",
187+
"labels": {
188+
"Extended": {},
189+
"NonHyperShiftHOST": {}
190+
},
191+
"resources": {
192+
"isolation": {}
193+
},
194+
"source": "openshift:payload:olmv1",
195+
"lifecycle": "informing",
196+
"environmentSelector": {
197+
"exclude": "topology==\"External\""
198+
}
199+
},
200+
{
201+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:78300-[Skipped:Disconnected]validation of container images using sigstore signatures with different policy [Serial][Disruptive][Slow]",
202+
"labels": {
203+
"Extended": {},
204+
"NonHyperShiftHOST": {}
205+
},
206+
"resources": {
207+
"isolation": {}
208+
},
209+
"source": "openshift:payload:olmv1",
210+
"lifecycle": "informing",
211+
"environmentSelector": {
212+
"exclude": "topology==\"External\""
213+
}
214+
},
215+
{
216+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:76983-[Skipped:Disconnected]install index and bundle from private image[Slow]",
217+
"labels": {
218+
"Extended": {},
219+
"NonHyperShiftHOST": {}
220+
},
221+
"resources": {
222+
"isolation": {}
223+
},
224+
"source": "openshift:payload:olmv1",
225+
"lifecycle": "informing",
226+
"environmentSelector": {
227+
"exclude": "topology==\"External\""
228+
}
229+
},
230+
{
231+
"name": "[sig-olmv1][Jira:OLM] clusterextension PolarionID:76985-[Skipped:Disconnected]authfile is updated automatically[Timeout:30m] [Serial][Disruptive][Slow]",
19232
"labels": {
20233
"Extended": {},
21234
"NonHyperShiftHOST": {}

‎openshift/tests-extension/cmd/main.go‎

Lines changed: 58 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
_ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test"
2424
_ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test/qe/specs"
2525
exutil "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test/qe/util"
26+
"github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test/qe/util/filters"
2627
)
2728

2829
func main() {
@@ -55,8 +56,7 @@ func main() {
5556
Name: "olmv1/parallel",
5657
Parents: []string{"openshift/conformance/parallel"},
5758
Qualifiers: []string{
58-
`((!labels.exists(l, l=="Extended")) || (labels.exists(l, l=="Extended") && labels.exists(l, l=="ReleaseGate"))) &&
59-
!(name.contains("[Serial]") || name.contains("[Slow]"))`,
59+
filters.BasedStandardTests(`!(name.contains("[Serial]") || name.contains("[Slow]"))`),
6060
},
6161
})
6262

@@ -69,8 +69,7 @@ func main() {
6969
Name: "olmv1/serial",
7070
Parents: []string{"openshift/conformance/serial"},
7171
Qualifiers: []string{
72-
`((!labels.exists(l, l=="Extended")) || (labels.exists(l, l=="Extended") && labels.exists(l, l=="ReleaseGate"))) &&
73-
(name.contains("[Serial]") && !name.contains("[Disruptive]") && !name.contains("[Slow]"))`,
72+
filters.BasedStandardTests(`(name.contains("[Serial]") && !name.contains("[Disruptive]") && !name.contains("[Slow]"))`),
7473
// refer to https://github.com/openshift/origin/blob/main/pkg/testsuites/standard_suites.go#L456
7574
},
7675
})
@@ -84,8 +83,7 @@ func main() {
8483
Name: "olmv1/slow",
8584
Parents: []string{"openshift/optional/slow"},
8685
Qualifiers: []string{
87-
`((!labels.exists(l, l=="Extended")) || (labels.exists(l, l=="Extended") && labels.exists(l, l=="ReleaseGate"))) &&
88-
name.contains("[Slow]")`,
86+
filters.BasedStandardTests(`name.contains("[Slow]")`),
8987
},
9088
})
9189

@@ -96,7 +94,7 @@ func main() {
9694
ext.AddSuite(e.Suite{
9795
Name: "olmv1/all",
9896
Qualifiers: []string{
99-
`(!labels.exists(l, l=="Extended")) || (labels.exists(l, l=="Extended") && labels.exists(l, l=="ReleaseGate"))`,
97+
filters.BasedStandardTests(``),
10098
},
10199
})
102100

@@ -105,7 +103,7 @@ func main() {
105103
ext.AddSuite(e.Suite{
106104
Name: "olmv1/extended",
107105
Qualifiers: []string{
108-
`labels.exists(l, l=="Extended")`,
106+
filters.BasedExtendedTests(``),
109107
},
110108
})
111109

@@ -114,7 +112,7 @@ func main() {
114112
ext.AddSuite(e.Suite{
115113
Name: "olmv1/extended/releasegate",
116114
Qualifiers: []string{
117-
`labels.exists(l, l=="Extended") && labels.exists(l, l=="ReleaseGate")`,
115+
filters.BasedExtendedReleaseGateTests(``),
118116
},
119117
})
120118

@@ -123,45 +121,88 @@ func main() {
123121
ext.AddSuite(e.Suite{
124122
Name: "olmv1/extended/candidate",
125123
Qualifiers: []string{
126-
`labels.exists(l, l=="Extended") && !labels.exists(l, l=="ReleaseGate")`,
124+
filters.BasedExtendedCandidateTests(``),
127125
},
128126
})
129127

128+
//
129+
// Categorization of Extended Candidate Tests:
130+
// ===========================================
131+
// The extended/candidate tests are categorized by test purpose and characteristics:
132+
//
133+
// 1. By Test Type:
134+
// - function: Functional tests that verify feature behavior and business logic
135+
// - stress: Stress tests that verify system behavior under resource pressure and load
136+
//
137+
// Relationship: candidate = function + stress + (other specialized test types)
138+
139+
// Extended Candidate Function Suite: Extended functional tests that don't meet OpenShift CI requirements
140+
// Contains extended tests that are not for openshift-tests and exclude stress tests
141+
ext.AddSuite(e.Suite{
142+
Name: "olmv1/extended/candidate/function",
143+
Qualifiers: []string{
144+
filters.BasedExtendedCandidateFuncTests(``),
145+
},
146+
})
147+
148+
//
149+
// Categorization of Extended Candidate Functional Tests:
150+
// =====================================================
151+
// The extended/candidate/function tests are categorized using two complementary approaches:
152+
//
153+
// 1. By Execution Model:
154+
// - parallel: Tests that can run concurrently (excludes [Serial] and [Slow])
155+
// - serial: Tests that must run one at a time ([Serial] but not [Slow])
156+
// - slow: Tests that take significant time to execute ([Slow])
157+
//
158+
// 2. By Execution Speed:
159+
// - fast: All non-slow functional tests (includes both parallel and serial, excludes [Slow])
160+
// - slow: Tests marked as [Slow] (same as above)
161+
//
162+
// Relationship: function = parallel + serial + slow = fast + slow
163+
130164
// Extended Candidate Suite Parallel Suite: extended tests that can run in parallel
131165
// Contains extended tests that can run concurrently (excludes Serial, Slow, and StressTest)
132166
ext.AddSuite(e.Suite{
133167
Name: "olmv1/extended/candidate/parallel",
134168
Qualifiers: []string{
135-
`(labels.exists(l, l=="Extended") && !labels.exists(l, l=="ReleaseGate") && !labels.exists(l, l=="StressTest")) &&
136-
!(name.contains("[Serial]") || name.contains("[Slow]"))`,
169+
filters.BasedExtendedCandidateFuncTests(`!(name.contains("[Serial]") || name.contains("[Slow]"))`),
137170
},
138171
})
139172
// Extended Candidate Serial Suite: extended tests that must run one at a time
140173
// Contains extended tests marked as [Serial] (includes Disruptive tests since not used for openshift-tests)
141174
ext.AddSuite(e.Suite{
142175
Name: "olmv1/extended/candidate/serial",
143176
Qualifiers: []string{
144-
`(labels.exists(l, l=="Extended") && !labels.exists(l, l=="ReleaseGate") && !labels.exists(l, l=="StressTest")) &&
145-
(name.contains("[Serial]") && !name.contains("[Slow]"))`,
177+
filters.BasedExtendedCandidateFuncTests(`(name.contains("[Serial]") && !name.contains("[Slow]"))`),
146178
// it is not used for openshift-tests, so it does not exclude Disruptive, so that we could use
147179
// olmv1/extended/candidate/serial to run all serial case including Disruptive cases
148180
},
149181
})
182+
183+
// Extended Candidate Fast Suite: extended functional tests excluding slow cases
184+
// Contains all extended functional tests that are not marked as [Slow] (includes both Serial and Parallel)
185+
// This provides a comprehensive functional test coverage with reasonable execution time
186+
ext.AddSuite(e.Suite{
187+
Name: "olmv1/extended/candidate/fast",
188+
Qualifiers: []string{
189+
filters.BasedExtendedCandidateFuncTests(`!name.contains("[Slow]")`),
190+
},
191+
})
150192
// Extended Candidate Slow Suite: extended tests that take significant time to run
151193
// Contains extended tests marked as [Slow] (long-running tests not suitable for fast CI)
152194
ext.AddSuite(e.Suite{
153195
Name: "olmv1/extended/candidate/slow",
154196
Qualifiers: []string{
155-
`(labels.exists(l, l=="Extended") && !labels.exists(l, l=="ReleaseGate") && !labels.exists(l, l=="StressTest")) &&
156-
name.contains("[Slow]")`,
197+
filters.BasedExtendedCandidateFuncTests(`name.contains("[Slow]")`),
157198
},
158199
})
159200
// Extended Candidate Stress Suite: extended stress tests
160201
// Contains extended tests designed for stress testing and resource exhaustion scenarios
161202
ext.AddSuite(e.Suite{
162203
Name: "olmv1/extended/candidate/stress",
163204
Qualifiers: []string{
164-
`labels.exists(l, l=="Extended") && !labels.exists(l, l=="ReleaseGate") && labels.exists(l, l=="StressTest")`,
205+
filters.BasedExtendedCandidateTests(`labels.exists(l, l=="StressTest")`),
165206
},
166207
})
167208

‎openshift/tests-extension/pkg/bindata/qe/bindata.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)