Skip to content

Collector scrape synchronization #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 28, 2025
Merged

Collector scrape synchronization #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 44 additions & 49 deletions collector/collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,51 +283,26 @@ func (e *Exporter) scheduledScrape(tick *time.Time) {

func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
e.totalScrapes.Inc()
var err error
var scrapemutex sync.Mutex
errChan := make(chan ScrapeResult, len(e.metricsToScrape.Metric))
errChan := make(chan error, len(e.metricsToScrape.Metric))
begun := time.Now()

defer func(begun time.Time) {
// other error
e.duration.Set(time.Since(begun).Seconds())
if err == nil {
e.error.Set(0)
} else {
e.error.Set(1)
}

// scrape error
close(errChan)
for scrape := range errChan {
if scrape.Err != nil {
if shouldLogScrapeError(scrape.Err, scrape.Metric.IgnoreZeroResult) {
level.Error(e.logger).Log("msg", "Error scraping metric",
"Context", scrape.Metric.Context,
"MetricsDesc", fmt.Sprint(scrape.Metric.MetricsDesc),
"time", time.Since(scrape.ScrapeStart),
"error", scrape.Err)
}
e.scrapeErrors.WithLabelValues(scrape.Metric.Context).Inc()
}
}

}(time.Now())

if err = e.db.Ping(); err != nil {
level.Debug(e.logger).Log("msg", "error = "+err.Error())
if strings.Contains(err.Error(), "sql: database is closed") {
if connectionError := e.db.Ping(); connectionError != nil {
level.Debug(e.logger).Log("msg", "error = "+connectionError.Error())
if strings.Contains(connectionError.Error(), "sql: database is closed") {
level.Info(e.logger).Log("msg", "Reconnecting to DB")
err = e.connect()
if err != nil {
level.Error(e.logger).Log("msg", "Error reconnecting to DB", err)
connectionError = e.connect()
if connectionError != nil {
level.Error(e.logger).Log("msg", "Error reconnecting to DB", connectionError)
}
}
}

if err = e.db.Ping(); err != nil {
if pingError := e.db.Ping(); pingError != nil {
level.Error(e.logger).Log("msg", "Error pinging oracle",
"error", err)
"error", pingError)
e.up.Set(0)
e.error.Set(1)
e.duration.Set(time.Since(begun).Seconds())
return
}

Expand All @@ -340,15 +315,10 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
e.reloadMetrics()
}

wg := sync.WaitGroup{}

for _, metric := range e.metricsToScrape.Metric {
wg.Add(1)
metric := metric //https://golang.org/doc/faq#closures_and_goroutines

go func() {
defer wg.Done()

level.Debug(e.logger).Log("msg", "About to scrape metric",
"Context", metric.Context,
"MetricsDesc", fmt.Sprint(metric.MetricsDesc),
Expand All @@ -360,11 +330,13 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
"Request", metric.Request)

if len(metric.Request) == 0 {
errChan <- errors.New("scrape request not found")
level.Error(e.logger).Log("msg", "Error scraping for "+fmt.Sprint(metric.MetricsDesc)+". Did you forget to define request in your toml file?")
return
}

if len(metric.MetricsDesc) == 0 {
errChan <- errors.New("metricsdesc not found")
level.Error(e.logger).Log("msg", "Error scraping for query"+fmt.Sprint(metric.Request)+". Did you forget to define metricsdesc in your toml file?")
return
}
Expand All @@ -373,19 +345,26 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
if metricType == "histogram" {
_, ok := metric.MetricsBuckets[column]
if !ok {
errChan <- errors.New("metricsbuckets not found")
level.Error(e.logger).Log("msg", "Unable to find MetricsBuckets configuration key for metric. (metric="+column+")")
return
}
}
}

scrapeStart := time.Now()
if err1 := func() error {
scrapemutex.Lock()
defer scrapemutex.Unlock()
return e.ScrapeMetric(e.db, ch, metric, tick)
}(); err1 != nil {
errChan <- ScrapeResult{Err: err1, Metric: metric, ScrapeStart: scrapeStart}
scrapeError := e.ScrapeMetric(e.db, ch, metric, tick)
// Always send the scrapeError, nil or non-nil
errChan <- scrapeError
if scrapeError != nil {
if shouldLogScrapeError(scrapeError, metric.IgnoreZeroResult) {
level.Error(e.logger).Log("msg", "Error scraping metric",
"Context", metric.Context,
"MetricsDesc", fmt.Sprint(metric.MetricsDesc),
"time", time.Since(scrapeStart),
"error", scrapeError)
}
e.scrapeErrors.WithLabelValues(metric.Context).Inc()
} else {
level.Debug(e.logger).Log("msg", "Successfully scraped metric",
"Context", metric.Context,
Expand All @@ -394,7 +373,23 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
}
}()
}
wg.Wait()

e.afterScrape(begun, len(e.metricsToScrape.Metric), errChan)
}

func (e *Exporter) afterScrape(begun time.Time, countMetrics int, errChan chan error) {
// Receive all scrape errors
totalErrors := 0.0
for i := 0; i < countMetrics; i++ {
scrapeError := <-errChan
if scrapeError != nil {
totalErrors++
}
}
close(errChan)

e.duration.Set(time.Since(begun).Seconds())
e.error.Set(totalErrors)
}

func (e *Exporter) connect() error {
Expand Down
20 changes: 20 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/UNO-SOFT/zlog v0.8.1 h1:TEFkGJHtUfTRgMkLZiAjLSHALjwSBdw6/zByMC5GJt4=
Expand All @@ -21,6 +23,7 @@ github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godror/godror v0.44.6 h1:3Aw45jpKkPsss80DVy4/XiqXUPCsHmW0efSHBQFi0h4=
github.com/godror/godror v0.44.6/go.mod h1:KJwMtQpK9o3WdEiNw7qvgSk827YDLj9MV/bXSzvUzlo=
Expand All @@ -40,10 +43,16 @@ github.com/godror/knownpb v0.1.2 h1:icMyYsYVpGmzhoVA01xyd0o4EaubR31JPK1UxQWe4kM=
github.com/godror/knownpb v0.1.2/go.mod h1:zs9hH+lwj7mnPHPnKCcxdOGz38Axa9uT+97Ng+Nnu5s=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
Expand All @@ -56,6 +65,10 @@ github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ=
github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
Expand Down Expand Up @@ -121,6 +134,8 @@ golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
Expand All @@ -146,12 +161,17 @@ golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down