diff --git a/.gitignore b/.gitignore index e18503cc..4f6e8992 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ collector-coverage.out dist/ .vscode/launch.json sample.yaml +.DS_Store diff --git a/README.md b/README.md index 657599fb..78e2cad5 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ This is a API testing tool. ## Features +* Multiple test report formats: Markdown, HTML, Stdout * Response Body fields equation check * Response Body [eval](https://expr.medv.io/) * Verify the Kubernetes resources diff --git a/cmd/run.go b/cmd/run.go index 6f3a75b8..56a4403b 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -46,7 +46,7 @@ func newDefaultRunOption() *runOption { } } -func newDiskCardRunOption() *runOption { +func newDiscardRunOption() *runOption { return &runOption{ reporter: runner.NewDiscardTestReporter(), reportWriter: runner.NewDiscardResultWriter(), @@ -74,7 +74,7 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`, flags.DurationVarP(&opt.duration, "duration", "", 0, "Running duration") flags.DurationVarP(&opt.requestTimeout, "request-timeout", "", time.Minute, "Timeout for per request") flags.BoolVarP(&opt.requestIgnoreError, "request-ignore-error", "", false, "Indicate if ignore the request error") - flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, discard, std") + flags.StringVarP(&opt.report, "report", "", "", "The type of target report. Supported: markdown, md, html, discard, std") flags.StringVarP(&opt.reportFile, "report-file", "", "", "The file path of the report") flags.BoolVarP(&opt.reportIgnore, "report-ignore", "", false, "Indicate if ignore the report output") flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution") @@ -98,6 +98,8 @@ func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) { switch o.report { case "markdown", "md": o.reportWriter = runner.NewMarkdownResultWriter(writer) + case "html": + o.reportWriter = runner.NewHTMLResultWriter(writer) case "discard": o.reportWriter = runner.NewDiscardResultWriter() case "", "std": @@ -178,7 +180,7 @@ func (o *runOption) runSuiteWithDuration(suite string) (err error) { defer sem.Release(1) defer wait.Done() defer func() { - fmt.Println("routing end with", time.Now().Sub(now)) + fmt.Println("routing end with", time.Since(now)) }() dataContext := getDefaultContext() diff --git a/cmd/run_test.go b/cmd/run_test.go index f82f9e7b..516eec20 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -53,7 +53,7 @@ func TestRunSuite(t *testing.T) { defer gock.Clean() util.MakeSureNotNil(tt.prepare)() ctx := getDefaultContext() - opt := newDiskCardRunOption() + opt := newDiscardRunOption() opt.requestTimeout = 30 * time.Second opt.limiter = limit.NewDefaultRateLimiter(0, 0) stopSingal := make(chan struct{}, 1) diff --git a/pkg/render/template.go b/pkg/render/template.go index ffbfadd8..b45a6a7f 100644 --- a/pkg/render/template.go +++ b/pkg/render/template.go @@ -2,7 +2,9 @@ package render import ( "bytes" + "fmt" "html/template" + "io" "strings" "github.com/Masterminds/sprig/v3" @@ -31,3 +33,12 @@ func FuncMap() template.FuncMap { } return funcs } + +// RenderThenPrint renders the template then prints the result +func RenderThenPrint(name, text string, ctx interface{}, w io.Writer) (err error) { + var report string + if report, err = Render(name, text, ctx); err == nil { + fmt.Fprint(w, report) + } + return +} diff --git a/pkg/render/template_test.go b/pkg/render/template_test.go index 98ed746d..19adc914 100644 --- a/pkg/render/template_test.go +++ b/pkg/render/template_test.go @@ -1,6 +1,7 @@ package render import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -56,3 +57,32 @@ func TestRender(t *testing.T) { }) } } + +func TestRenderThenPrint(t *testing.T) { + tests := []struct { + name string + tplText string + ctx interface{} + buf *bytes.Buffer + expect string + }{{ + name: "simple", + tplText: `{{max 1 2 3}}`, + ctx: nil, + buf: new(bytes.Buffer), + expect: `3`, + }, { + name: "with a map as context", + tplText: `{{.name}}`, + ctx: map[string]string{"name": "linuxsuren"}, + buf: new(bytes.Buffer), + expect: "linuxsuren", + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := RenderThenPrint(tt.name, tt.tplText, tt.ctx, tt.buf) + assert.NoError(t, err) + assert.Equal(t, tt.expect, tt.buf.String()) + }) + } +} diff --git a/pkg/runner/data/html.html b/pkg/runner/data/html.html new file mode 100644 index 00000000..18213163 --- /dev/null +++ b/pkg/runner/data/html.html @@ -0,0 +1,33 @@ + + + + API Testing Report + + + + + + + + {{- range $val := .}} + + {{- end}} +
API Testing Report
APIAverageMaxMinCountError
{{$val.API}}{{$val.Average}}{{$val.Max}}{{$val.Min}}{{$val.Count}}{{$val.Error}}
+ + + diff --git a/pkg/runner/reporter_memory.go b/pkg/runner/reporter_memory.go index 06e34632..2c6fbd7b 100644 --- a/pkg/runner/reporter_memory.go +++ b/pkg/runner/reporter_memory.go @@ -58,12 +58,8 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice, item.Total += duration item.Count += 1 - if record.EndTime.After(item.Last) { - item.Last = record.EndTime - } - if record.BeginTime.Before(item.First) { - item.First = record.BeginTime - } + item.Last = getLaterTime(record.EndTime, item.Last) + item.LastErrorMessage = getOriginalStringWhenEmpty(item.LastErrorMessage, record.GetErrorMessage()) } else { resultWithTotal[api] = &ReportResultWithTotal{ ReportResult: ReportResult{ @@ -77,6 +73,7 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice, Last: record.EndTime, Total: duration, } + resultWithTotal[api].LastErrorMessage = record.GetErrorMessage() } } @@ -91,3 +88,17 @@ func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice, sort.Sort(result) return } + +func getLaterTime(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} + +func getOriginalStringWhenEmpty(a, b string) string { + if b == "" { + return a + } + return b +} diff --git a/pkg/runner/reporter_memory_test.go b/pkg/runner/reporter_memory_test.go index 14932ada..c6215144 100644 --- a/pkg/runner/reporter_memory_test.go +++ b/pkg/runner/reporter_memory_test.go @@ -38,6 +38,7 @@ func TestExportAllReportResults(t *testing.T) { BeginTime: now, EndTime: now.Add(time.Second * 4), Error: errors.New("fake"), + Body: "fake", }, { API: urlFoo, Method: http.MethodGet, @@ -62,12 +63,13 @@ func TestExportAllReportResults(t *testing.T) { Count: 1, Error: 0, }, { - API: "GET http://foo", - Average: time.Second * 3, - Max: time.Second * 4, - Min: time.Second * 2, - Count: 3, - Error: 1, + API: "GET http://foo", + Average: time.Second * 3, + Max: time.Second * 4, + Min: time.Second * 2, + Count: 3, + Error: 1, + LastErrorMessage: "fake", }, { API: "GET http://bar", Average: time.Second, @@ -77,6 +79,25 @@ func TestExportAllReportResults(t *testing.T) { Count: 1, Error: 0, }}, + }, { + name: "first record has error", + records: []*runner.ReportRecord{{ + API: urlFoo, + Method: http.MethodGet, + BeginTime: now, + EndTime: now.Add(time.Second * 4), + Error: errors.New("fake"), + Body: "fake", + }}, + expect: runner.ReportResultSlice{{ + API: "GET http://foo", + Average: time.Second * 4, + Max: time.Second * 4, + Min: time.Second * 4, + Count: 1, + Error: 1, + LastErrorMessage: "fake", + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/runner/simple.go b/pkg/runner/simple.go index f6a0da6b..8006e39b 100644 --- a/pkg/runner/simple.go +++ b/pkg/runner/simple.go @@ -102,6 +102,15 @@ func (r *ReportRecord) ErrorCount() int { return 1 } +// GetErrorMessage returns the error message +func (r *ReportRecord) GetErrorMessage() string { + if r.ErrorCount() > 0 { + return r.Body + } else { + return "" + } +} + // NewReportRecord creates a record, and set the begin time to be now func NewReportRecord() *ReportRecord { return &ReportRecord{ @@ -111,13 +120,14 @@ func NewReportRecord() *ReportRecord { // ReportResult represents the report result of a set of the same API requests type ReportResult struct { - API string - Count int - Average time.Duration - Max time.Duration - Min time.Duration - QPS int - Error int + API string + Count int + Average time.Duration + Max time.Duration + Min time.Duration + QPS int + Error int + LastErrorMessage string } // ReportResultSlice is the alias type of ReportResult slice @@ -202,10 +212,6 @@ func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataConte }(record) defer func() { - if testcase.Clean.CleanPrepare { - err = r.doCleanPrepare(testcase) - } - if err == nil { err = runJob(testcase.After) } diff --git a/pkg/runner/simple_test.go b/pkg/runner/simple_test.go index 4132de51..ba9cb3e2 100644 --- a/pkg/runner/simple_test.go +++ b/pkg/runner/simple_test.go @@ -72,9 +72,6 @@ func TestTestCase(t *testing.T) { Before: atest.Job{ Items: []string{"sleep(1)"}, }, - Clean: atest.Clean{ - CleanPrepare: true, - }, }, execer: fakeruntime.FakeExecer{}, prepare: func() { diff --git a/pkg/runner/testdata/report.html b/pkg/runner/testdata/report.html new file mode 100644 index 00000000..1686a654 --- /dev/null +++ b/pkg/runner/testdata/report.html @@ -0,0 +1,31 @@ + + + + API Testing Report + + + + + + + + +
API Testing Report
APIAverageMaxMinCountError
/foo3ns3ns3ns10
+ + + \ No newline at end of file diff --git a/pkg/runner/writer_html.go b/pkg/runner/writer_html.go new file mode 100644 index 00000000..a1d181cf --- /dev/null +++ b/pkg/runner/writer_html.go @@ -0,0 +1,25 @@ +package runner + +import ( + _ "embed" + "io" + + "github.com/linuxsuren/api-testing/pkg/render" +) + +type htmlResultWriter struct { + writer io.Writer +} + +// NewHTMLResultWriter creates a new htmlResultWriter +func NewHTMLResultWriter(writer io.Writer) ReportResultWriter { + return &htmlResultWriter{writer: writer} +} + +// Output writes the HTML base report to target writer +func (w *htmlResultWriter) Output(result []ReportResult) (err error) { + return render.RenderThenPrint("html-report", htmlReport, result, w.writer) +} + +//go:embed data/html.html +var htmlReport string diff --git a/pkg/runner/writer_html_test.go b/pkg/runner/writer_html_test.go new file mode 100644 index 00000000..34135543 --- /dev/null +++ b/pkg/runner/writer_html_test.go @@ -0,0 +1,43 @@ +package runner_test + +import ( + "bytes" + "testing" + + _ "embed" + + "github.com/linuxsuren/api-testing/pkg/runner" + "github.com/stretchr/testify/assert" +) + +func TestHTMLResultWriter(t *testing.T) { + tests := []struct { + name string + buf *bytes.Buffer + results []runner.ReportResult + expect string + }{{ + name: "simple", + buf: new(bytes.Buffer), + results: []runner.ReportResult{{ + API: "/foo", + Max: 3, + Min: 3, + Average: 3, + Error: 0, + Count: 1, + }}, + expect: htmlReportExpect, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := runner.NewHTMLResultWriter(tt.buf) + err := w.Output(tt.results) + assert.NoError(t, err) + assert.Equal(t, tt.expect, tt.buf.String()) + }) + } +} + +//go:embed testdata/report.html +var htmlReportExpect string diff --git a/pkg/runner/writer_markdown.go b/pkg/runner/writer_markdown.go new file mode 100644 index 00000000..69d24f5f --- /dev/null +++ b/pkg/runner/writer_markdown.go @@ -0,0 +1,25 @@ +package runner + +import ( + _ "embed" + "io" + + "github.com/linuxsuren/api-testing/pkg/render" +) + +type markdownResultWriter struct { + writer io.Writer +} + +// NewMarkdownResultWriter creates the Markdown writer +func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter { + return &markdownResultWriter{writer: writer} +} + +// Output writes the Markdown based report to target writer +func (w *markdownResultWriter) Output(result []ReportResult) (err error) { + return render.RenderThenPrint("md-report", markdownReport, result, w.writer) +} + +//go:embed data/report.md +var markdownReport string diff --git a/pkg/runner/writer_markdown_test.go b/pkg/runner/writer_markdown_test.go new file mode 100644 index 00000000..1a152f90 --- /dev/null +++ b/pkg/runner/writer_markdown_test.go @@ -0,0 +1,35 @@ +package runner_test + +import ( + "bytes" + "testing" + + "github.com/linuxsuren/api-testing/pkg/runner" + "github.com/stretchr/testify/assert" +) + +func TestMarkdownWriter(t *testing.T) { + buf := new(bytes.Buffer) + writer := runner.NewMarkdownResultWriter(buf) + + err := writer.Output([]runner.ReportResult{{ + API: "api", + Average: 3, + Max: 4, + Min: 2, + Count: 3, + Error: 0, + }, { + API: "api", + Average: 3, + Max: 4, + Min: 2, + Count: 3, + Error: 0, + }}) + assert.Nil(t, err) + assert.Equal(t, `| API | Average | Max | Min | Count | Error | +|---|---|---|---|---|---| +| api | 3ns | 4ns | 2ns | 3 | 0 | +| api | 3ns | 4ns | 2ns | 3 | 0 |`, buf.String()) +} diff --git a/pkg/runner/writer_std.go b/pkg/runner/writer_std.go index da5a0522..2a34bd0d 100644 --- a/pkg/runner/writer_std.go +++ b/pkg/runner/writer_std.go @@ -1,11 +1,9 @@ package runner import ( - "bytes" _ "embed" "fmt" "io" - "text/template" ) type stdResultWriter struct { @@ -23,36 +21,19 @@ func NewDiscardResultWriter() ReportResultWriter { } // Output writer the report to target writer -func (w *stdResultWriter) Output(result []ReportResult) error { +func (w *stdResultWriter) Output(results []ReportResult) error { + var errResults []ReportResult fmt.Fprintf(w.writer, "API Average Max Min QPS Count Error\n") - for _, r := range result { + for _, r := range results { fmt.Fprintf(w.writer, "%s %v %v %v %d %d %d\n", r.API, r.Average, r.Max, r.Min, r.QPS, r.Count, r.Error) + if r.Error > 0 && r.LastErrorMessage != "" { + errResults = append(errResults, r) + } } - return nil -} - -type markdownResultWriter struct { - writer io.Writer -} -// NewMarkdownResultWriter creates the Markdown writer -func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter { - return &markdownResultWriter{writer: writer} -} - -// Output writer the Markdown based report to target writer -func (w *markdownResultWriter) Output(result []ReportResult) (err error) { - var tpl *template.Template - if tpl, err = template.New("report").Parse(markDownReport); err == nil { - buf := new(bytes.Buffer) - - if err = tpl.Execute(buf, result); err == nil { - fmt.Fprint(w.writer, buf.String()) - } + for _, r := range errResults { + fmt.Fprintf(w.writer, "%s error: %s\n", r.API, r.LastErrorMessage) } - return + return nil } - -//go:embed data/report.md -var markDownReport string diff --git a/pkg/runner/writer_std_test.go b/pkg/runner/writer_std_test.go index 5701b77d..f7d2c40e 100644 --- a/pkg/runner/writer_std_test.go +++ b/pkg/runner/writer_std_test.go @@ -8,33 +8,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMarkdownWriter(t *testing.T) { - buf := new(bytes.Buffer) - writer := runner.NewMarkdownResultWriter(buf) - - err := writer.Output([]runner.ReportResult{{ - API: "api", - Average: 3, - Max: 4, - Min: 2, - Count: 3, - Error: 0, - }, { - API: "api", - Average: 3, - Max: 4, - Min: 2, - Count: 3, - Error: 0, - }}) - assert.Nil(t, err) - assert.Equal(t, `| API | Average | Max | Min | Count | Error | -|---|---|---|---|---|---| -| api | 3ns | 4ns | 2ns | 3 | 0 | -| api | 3ns | 4ns | 2ns | 3 | 0 | -`, buf.String()) -} - func TestNewStdResultWriter(t *testing.T) { tests := []struct { name string @@ -61,6 +34,39 @@ func TestNewStdResultWriter(t *testing.T) { }}, expect: `API Average Max Min QPS Count Error api 1ns 1ns 1ns 10 1 0 +`, + }, { + name: "have errors", + buf: new(bytes.Buffer), + results: []runner.ReportResult{{ + API: "api", + Average: 1, + Max: 1, + Min: 1, + QPS: 10, + Count: 1, + Error: 1, + LastErrorMessage: "error", + }}, + expect: `API Average Max Min QPS Count Error +api 1ns 1ns 1ns 10 1 1 +api error: error +`, + }, { + name: "have no errors but with message", + buf: new(bytes.Buffer), + results: []runner.ReportResult{{ + API: "api", + Average: 1, + Max: 1, + Min: 1, + QPS: 10, + Count: 1, + Error: 0, + LastErrorMessage: "message", + }}, + expect: `API Average Max Min QPS Count Error +api 1ns 1ns 1ns 10 1 0 `, }} for _, tt := range tests { diff --git a/pkg/testing/case.go b/pkg/testing/case.go index d0138a56..f1037621 100644 --- a/pkg/testing/case.go +++ b/pkg/testing/case.go @@ -15,7 +15,6 @@ type TestCase struct { After Job `yaml:"after,omitempty" json:"after"` Request Request `yaml:"request" json:"request"` Expect Response `yaml:"expect,omitempty" json:"expect"` - Clean Clean `yaml:"clean,omitempty" json:"-"` } // InScope returns true if the test case is in scope with the given items. @@ -57,8 +56,3 @@ type Response struct { Verify []string `yaml:"verify,omitempty" json:"verify,omitempty"` Schema string `yaml:"schema,omitempty" json:"schema,omitempty"` } - -// Clean represents the clean work after testing -type Clean struct { - CleanPrepare bool `yaml:"cleanPrepare"` -}