From be4864ed5aeca29b017d94e5de216846997bdc4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 13:06:10 +0000 Subject: [PATCH] Bump github.com/felixge/fgprof from 0.9.2 to 0.9.3 Bumps [github.com/felixge/fgprof](https://github.com/felixge/fgprof) from 0.9.2 to 0.9.3. - [Release notes](https://github.com/felixge/fgprof/releases) - [Commits](https://github.com/felixge/fgprof/compare/v0.9.2...v0.9.3) --- updated-dependencies: - dependency-name: github.com/felixge/fgprof dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- vendor/github.com/felixge/fgprof/README.md | 15 +- vendor/github.com/felixge/fgprof/fgprof.go | 225 ++++++++++++++++++--- vendor/github.com/felixge/fgprof/format.go | 105 ---------- vendor/github.com/felixge/fgprof/pprof.go | 56 ----- vendor/modules.txt | 2 +- 7 files changed, 202 insertions(+), 207 deletions(-) delete mode 100644 vendor/github.com/felixge/fgprof/format.go delete mode 100644 vendor/github.com/felixge/fgprof/pprof.go diff --git a/go.mod b/go.mod index 63b2f0e7b94..1f66f75be4a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/dustin/go-humanize v1.0.0 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb - github.com/felixge/fgprof v0.9.2 + github.com/felixge/fgprof v0.9.3 github.com/go-kit/log v0.2.1 github.com/go-openapi/strfmt v0.21.3 github.com/go-openapi/swag v0.22.3 diff --git a/go.sum b/go.sum index 7a6933a402c..da3b3dd45a3 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc= -github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= diff --git a/vendor/github.com/felixge/fgprof/README.md b/vendor/github.com/felixge/fgprof/README.md index 1c1b20536fe..15b16794b95 100644 --- a/vendor/github.com/felixge/fgprof/README.md +++ b/vendor/github.com/felixge/fgprof/README.md @@ -10,6 +10,8 @@ Go's builtin sampling CPU profiler can only show On-CPU time, but it's better th fgprof is designed for analyzing applications with mixed I/O and CPU workloads. This kind of profiling is also known as wall-clock profiling. +⚠️ Please upgrade to Go 1.19 or newer. In older versions of Go fgprof can cause significant STW latencies in applications with a lot of goroutines (> 1-10k). See [CL 387415](https://go-review.googlesource.com/c/go/+/387415) for more details. + ## Quick Start If this is the first time you hear about fgprof, you should start by reading about [The Problem](#the-problem) & [How it Works](#how-it-works). @@ -175,7 +177,7 @@ fgprof is implemented as a background goroutine that wakes up 99 times per secon This data is used to maintain an in-memory stack counter which can be converted to the pprof or folded output format. The meat of the implementation is super simple and < 100 lines of code, you should [check it out](./fgprof.go). -The overhead of fgprof increases with the number of active goroutines (including those waiting on I/O, Channels, Locks, etc.) executed by your program. If your program typically has less than 1000 active goroutines, you shouldn't have much to worry about. However, at 10k or more goroutines fgprof is likely to become unable to maintain its sampling rate and to significantly degrade the performance of your application, see [BenchmarkProfilerGoroutines.txt](./BenchmarkProfilerGoroutines.txt). The latter is due to `runtime.GoroutineProfile()` calling `stopTheWorld()`. For now the advise is to test the impact of the profiler on a development environment before running it against production instances. There are ideas for making fgprof more scalable and safe for programs with a high number of goroutines, but they will likely need improved APIs from the Go core. +The overhead of fgprof increases with the number of active goroutines (including those waiting on I/O, Channels, Locks, etc.) executed by your program. If your program typically has less than 1000 active goroutines, you shouldn't have much to worry about. However, at 10k or more goroutines fgprof might start to cause some noticeable overhead. ### Go's builtin CPU Profiler @@ -183,21 +185,10 @@ The builtin Go CPU profiler uses the [setitimer(2)](https://linux.die.net/man/2/ Since Go uses non-blocking I/O, Goroutines that wait on I/O are parked and not running on any threads. Therefore they end up being largely invisible to Go's builtin CPU profiler. -## The Future of Go Profiling - -There is a great proposal for [hardware performance counters for CPU profiling](https://go.googlesource.com/proposal/+/refs/changes/08/219508/2/design/36821-perf-counter-pprof.md#5-empirical-evidence-on-the-accuracy-and-precision-of-pmu-profiles) in Go. The proposal is aimed at making the builtin CPU Profiler even more accurate, especially under highly parallel workloads on many CPUs. It also includes a very in-depth analysis of the current profiler. Based on the design, I think the proposed profiler would also be blind to I/O workloads, but still seems appealing for CPU based workloads. - -As far as fgprof itself is concerned, I might implement streaming output, leaving the final aggregation to other tools. This would open the door to even more advanced analysis, perhaps by integrating with tools such as [flamescope](https://github.com/Netflix/flamescope). - -Additionally I'm also open to the idea of contributing fgprof to the Go project itself. I've [floated the idea](https://groups.google.com/g/golang-dev/c/LCJyvL90xv8) on the golang-dev mailing list, so let's see what happens. - - ## Known Issues There is no perfect approach to profiling, and fgprof is no exception. Below is a list of known issues that will hopefully not be of practical concern for most users, but are important to highlight. -- fgprof can't catch goroutines while they are running in loops without function calls, only when they get asynchronously preempted. This can lead to reporting inaccuracies. Use the builtin CPU profiler if this is a problem for you. -- fgprof may not work in Go 1.13 if another goroutine is in a loop without function calls the whole time. Async preemption in Go 1.14 should mostly fix this issue. - Internal C functions are not showing up in the stack traces, e.g. `runtime.nanotime` which is called by `time.Since` in the example program. - The current implementation is relying on the Go scheduler to schedule the internal goroutine at a fixed sample rate. Scheduler delays, especially biased ones, might cause inaccuracies. diff --git a/vendor/github.com/felixge/fgprof/fgprof.go b/vendor/github.com/felixge/fgprof/fgprof.go index 31871c8395a..18f7d6f5fe4 100644 --- a/vendor/github.com/felixge/fgprof/fgprof.go +++ b/vendor/github.com/felixge/fgprof/fgprof.go @@ -4,16 +4,34 @@ package fgprof import ( + "fmt" "io" "runtime" + "sort" "strings" "time" + + "github.com/google/pprof/profile" +) + +// Format decides how the output is rendered to the user. +type Format string + +const ( + // FormatFolded is used by Brendan Gregg's FlameGraph utility, see + // https://github.com/brendangregg/FlameGraph#2-fold-stacks. + FormatFolded Format = "folded" + // FormatPprof is used by Google's pprof utility, see + // https://github.com/google/pprof/blob/master/proto/README.md. + FormatPprof Format = "pprof" ) // Start begins profiling the goroutines of the program and returns a function // that needs to be invoked by the caller to stop the profiling and write the // results to w using the given format. func Start(w io.Writer, format Format) func() error { + startTime := time.Now() + // Go's CPU profiler uses 100hz, but 99hz might be less likely to result in // accidental synchronization with the program we're profiling. const hz = 99 @@ -21,7 +39,7 @@ func Start(w io.Writer, format Format) func() error { stopCh := make(chan struct{}) prof := &profiler{} - stackCounts := stackCounter{} + profile := newWallclockProfile() go func() { defer ticker.Stop() @@ -30,7 +48,7 @@ func Start(w io.Writer, format Format) func() error { select { case <-ticker.C: stacks := prof.GoroutineProfile() - stackCounts.Update(stacks) + profile.Add(stacks) case <-stopCh: return } @@ -39,7 +57,9 @@ func Start(w io.Writer, format Format) func() error { return func() error { stopCh <- struct{}{} - return writeFormat(w, stackCounts.HumanMap(prof.SelfFrame()), format, hz) + endTime := time.Now() + profile.Ignore(prof.SelfFrames()...) + return profile.Export(w, format, hz, startTime, endTime) } } @@ -85,58 +105,203 @@ func (p *profiler) GoroutineProfile() []runtime.StackRecord { } } -func (p *profiler) SelfFrame() *runtime.Frame { - return p.selfFrame +// SelfFrames returns frames that belong to the profiler so that we can ignore +// them when exporting the final profile. +func (p *profiler) SelfFrames() []*runtime.Frame { + if p.selfFrame != nil { + return []*runtime.Frame{p.selfFrame} + } + return nil } -type stringStackCounter map[string]int +func newWallclockProfile() *wallclockProfile { + return &wallclockProfile{stacks: map[[32]uintptr]*wallclockStack{}} +} -func (s stringStackCounter) Update(p []runtime.StackRecord) { - for _, pp := range p { - frames := runtime.CallersFrames(pp.Stack()) +// wallclockProfile holds a wallclock profile that can be exported in different +// formats. +type wallclockProfile struct { + stacks map[[32]uintptr]*wallclockStack + ignore []*runtime.Frame +} - var stack []string - for { - frame, more := frames.Next() - stack = append([]string{frame.Function}, stack...) - if !more { - break +// wallclockStack holds the symbolized frames of a stack trace and the number +// of times it has been seen. +type wallclockStack struct { + frames []*runtime.Frame + count int +} + +// Ignore sets a list of frames that should be ignored when exporting the +// profile. +func (p *wallclockProfile) Ignore(frames ...*runtime.Frame) { + p.ignore = frames +} + +// Add adds the given stack traces to the profile. +func (p *wallclockProfile) Add(stackRecords []runtime.StackRecord) { + for _, stackRecord := range stackRecords { + if _, ok := p.stacks[stackRecord.Stack0]; !ok { + ws := &wallclockStack{} + // symbolize pcs into frames + frames := runtime.CallersFrames(stackRecord.Stack()) + for { + frame, more := frames.Next() + ws.frames = append(ws.frames, &frame) + if !more { + break + } + } + p.stacks[stackRecord.Stack0] = ws + } + p.stacks[stackRecord.Stack0].count++ + } +} + +func (p *wallclockProfile) Export(w io.Writer, f Format, hz int, startTime, endTime time.Time) error { + switch f { + case FormatFolded: + return p.exportFolded(w) + case FormatPprof: + return p.exportPprof(hz, startTime, endTime).Write(w) + default: + return fmt.Errorf("unknown format: %q", f) + } +} + +// exportStacks returns the stacks in this profile except those that have been +// set to Ignore(). +func (p *wallclockProfile) exportStacks() []*wallclockStack { + stacks := make([]*wallclockStack, 0, len(p.stacks)) +nextStack: + for _, ws := range p.stacks { + for _, f := range ws.frames { + for _, igf := range p.ignore { + if f.Entry == igf.Entry { + continue nextStack + } } } - key := strings.Join(stack, ";") - s[key]++ + stacks = append(stacks, ws) } + return stacks } -type stackCounter map[[32]uintptr]int +func (p *wallclockProfile) exportFolded(w io.Writer) error { + var lines []string + stacks := p.exportStacks() + for _, ws := range stacks { + var foldedStack []string + for _, f := range ws.frames { + foldedStack = append(foldedStack, f.Function) + } + line := fmt.Sprintf("%s %d", strings.Join(foldedStack, ";"), ws.count) + lines = append(lines, line) + } + sort.Strings(lines) + _, err := io.WriteString(w, strings.Join(lines, "\n")+"\n") + return err +} + +func (p *wallclockProfile) exportPprof(hz int, startTime, endTime time.Time) *profile.Profile { + prof := &profile.Profile{} + m := &profile.Mapping{ID: 1, HasFunctions: true} + prof.Period = int64(1e9 / hz) // Number of nanoseconds between samples. + prof.TimeNanos = startTime.UnixNano() + prof.DurationNanos = int64(endTime.Sub(startTime)) + prof.Mapping = []*profile.Mapping{m} + prof.SampleType = []*profile.ValueType{ + { + Type: "samples", + Unit: "count", + }, + { + Type: "time", + Unit: "nanoseconds", + }, + } + prof.PeriodType = &profile.ValueType{ + Type: "wallclock", + Unit: "nanoseconds", + } + + type functionKey struct { + Name string + Filename string + } + funcIdx := map[functionKey]*profile.Function{} -func (s stackCounter) Update(p []runtime.StackRecord) { - for _, pp := range p { - s[pp.Stack0]++ + type locationKey struct { + Function functionKey + Line int } + locationIdx := map[locationKey]*profile.Location{} + for _, ws := range p.exportStacks() { + sample := &profile.Sample{ + Value: []int64{ + int64(ws.count), + int64(1000 * 1000 * 1000 / hz * ws.count), + }, + } + + for _, frame := range ws.frames { + fnKey := functionKey{Name: frame.Function, Filename: frame.File} + function, ok := funcIdx[fnKey] + if !ok { + function = &profile.Function{ + ID: uint64(len(prof.Function)) + 1, + Name: frame.Function, + SystemName: frame.Function, + Filename: frame.File, + } + funcIdx[fnKey] = function + prof.Function = append(prof.Function, function) + } + + locKey := locationKey{Function: fnKey, Line: frame.Line} + location, ok := locationIdx[locKey] + if !ok { + location = &profile.Location{ + ID: uint64(len(prof.Location)) + 1, + Mapping: m, + Line: []profile.Line{{ + Function: function, + Line: int64(frame.Line), + }}, + } + locationIdx[locKey] = location + prof.Location = append(prof.Location, location) + } + sample.Location = append(sample.Location, location) + } + prof.Sample = append(prof.Sample, sample) + } + return prof } -// @TODO(fg) create a better interface that avoids the pprof output having to -// split the stacks using the `;` separator. -func (s stackCounter) HumanMap(exclude *runtime.Frame) map[string]int { - m := map[string]int{} +type symbolizedStacks map[[32]uintptr][]frameCount + +func (w wallclockProfile) Symbolize(exclude *runtime.Frame) symbolizedStacks { + m := make(symbolizedStacks) outer: - for stack0, count := range s { + for stack0, ws := range w.stacks { frames := runtime.CallersFrames((&runtime.StackRecord{Stack0: stack0}).Stack()) - var stack []string for { frame, more := frames.Next() if frame.Entry == exclude.Entry { continue outer } - stack = append([]string{frame.Function}, stack...) + m[stack0] = append(m[stack0], frameCount{Frame: &frame, Count: ws.count}) if !more { break } } - key := strings.Join(stack, ";") - m[key] = count } return m } + +type frameCount struct { + *runtime.Frame + Count int +} diff --git a/vendor/github.com/felixge/fgprof/format.go b/vendor/github.com/felixge/fgprof/format.go deleted file mode 100644 index c9bfe9fd2e9..00000000000 --- a/vendor/github.com/felixge/fgprof/format.go +++ /dev/null @@ -1,105 +0,0 @@ -package fgprof - -import ( - "fmt" - "io" - "sort" - "strings" - - "github.com/google/pprof/profile" -) - -// Format decides how the ouput is rendered to the user. -type Format string - -const ( - // FormatFolded is used by Brendan Gregg's FlameGraph utility, see - // https://github.com/brendangregg/FlameGraph#2-fold-stacks. - FormatFolded Format = "folded" - // FormatPprof is used by Google's pprof utility, see - // https://github.com/google/pprof/blob/master/proto/README.md. - FormatPprof Format = "pprof" -) - -func writeFormat(w io.Writer, s map[string]int, f Format, hz int) error { - switch f { - case FormatFolded: - return writeFolded(w, s) - case FormatPprof: - return toPprof(s, hz).Write(w) - default: - return fmt.Errorf("unknown format: %q", f) - } -} - -func writeFolded(w io.Writer, s map[string]int) error { - for _, stack := range sortedKeys(s) { - count := s[stack] - if _, err := fmt.Fprintf(w, "%s %d\n", stack, count); err != nil { - return err - } - } - return nil -} - -func toPprof(s map[string]int, hz int) *profile.Profile { - functionID := uint64(1) - locationID := uint64(1) - line := int64(1) - - p := &profile.Profile{} - m := &profile.Mapping{ID: 1, HasFunctions: true} - p.Mapping = []*profile.Mapping{m} - p.SampleType = []*profile.ValueType{ - { - Type: "samples", - Unit: "count", - }, - { - Type: "time", - Unit: "nanoseconds", - }, - } - - for stack, count := range s { - sample := &profile.Sample{ - Value: []int64{ - int64(count), - int64(1000 * 1000 * 1000 / hz * count), - }, - } - for _, fnName := range strings.Split(stack, ";") { - function := &profile.Function{ - ID: functionID, - Name: fnName, - } - p.Function = append(p.Function, function) - - location := &profile.Location{ - ID: locationID, - Mapping: m, - Line: []profile.Line{{Function: function}}, - } - p.Location = append(p.Location, location) - sample.Location = append([]*profile.Location{location}, sample.Location...) - - line++ - - locationID++ - functionID++ - } - p.Sample = append(p.Sample, sample) - } - return p -} - -func sortedKeys(s map[string]int) []string { - keys := make([]string, len(s)) - i := 0 - for stack := range s { - keys[i] = stack - i++ - } - sort.Strings(keys) - return keys -} diff --git a/vendor/github.com/felixge/fgprof/pprof.go b/vendor/github.com/felixge/fgprof/pprof.go deleted file mode 100644 index 4312ea7157d..00000000000 --- a/vendor/github.com/felixge/fgprof/pprof.go +++ /dev/null @@ -1,56 +0,0 @@ -package fgprof - -import ( - "strings" - - "github.com/google/pprof/profile" -) - -func toProfile(s map[string]int, hz int) *profile.Profile { - functionID := uint64(1) - locationID := uint64(1) - - p := &profile.Profile{} - m := &profile.Mapping{ID: 1, HasFunctions: true} - p.Mapping = []*profile.Mapping{m} - p.SampleType = []*profile.ValueType{ - { - Type: "samples", - Unit: "count", - }, - { - Type: "time", - Unit: "nanoseconds", - }, - } - - for _, stack := range sortedKeys(s) { - count := s[stack] - sample := &profile.Sample{ - Value: []int64{ - int64(count), - int64(1000 * 1000 * 1000 / hz * count), - }, - } - for _, fnName := range strings.Split(stack, ";") { - function := &profile.Function{ - ID: functionID, - Name: fnName, - } - p.Function = append(p.Function, function) - - location := &profile.Location{ - ID: locationID, - Mapping: m, - Line: []profile.Line{{Function: function}}, - } - p.Location = append(p.Location, location) - sample.Location = append(sample.Location, location) - - locationID++ - functionID++ - } - p.Sample = append(p.Sample, sample) - } - return p -} diff --git a/vendor/modules.txt b/vendor/modules.txt index eb936cc0252..8c71dc76b26 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -273,7 +273,7 @@ github.com/facette/natsort # github.com/fatih/color v1.13.0 ## explicit; go 1.13 github.com/fatih/color -# github.com/felixge/fgprof v0.9.2 +# github.com/felixge/fgprof v0.9.3 ## explicit; go 1.14 github.com/felixge/fgprof # github.com/felixge/httpsnoop v1.0.3