From 2abb5c9bcd27c4090ca689fc23ab4d5daf77d1d4 Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Tue, 3 Jun 2025 08:42:14 +0000 Subject: [PATCH 1/6] Add a collector for `pg_buffercache`. We use this code in production in a internal fork. We would like to push it upstream. Default it to off, because it requires an extension to be loaded. Signed-off-by: Peter Nuttall --- collector/pg_buffercache.go | 143 +++++++++++++++++++++++++++++++ collector/pg_buffercache_test.go | 73 ++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 collector/pg_buffercache.go create mode 100644 collector/pg_buffercache_test.go diff --git a/collector/pg_buffercache.go b/collector/pg_buffercache.go new file mode 100644 index 000000000..8de1329aa --- /dev/null +++ b/collector/pg_buffercache.go @@ -0,0 +1,143 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "database/sql" + "log/slog" + + "github.com/blang/semver/v4" + "github.com/prometheus/client_golang/prometheus" +) + +const buffercacheSubsystem = "buffercache" + +func init() { + registerCollector(buffercacheSubsystem, defaultDisabled, NewBuffercacheCollector) +} + +// BuffercacheCollector collects stats from pg_buffercache: https://www.postgresql.org/docs/current/pgbuffercache.html. +// +// It depends on the extension being loaded with +// +// create extension pg_buffercache; +// +// It does not take locks, see the PG docs above. +type BuffercacheCollector struct { + log *slog.Logger +} + +func NewBuffercacheCollector(config collectorConfig) (Collector, error) { + return &BuffercacheCollector{ + log: config.logger, + }, nil +} + +var ( + buffersUsedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_used"), + "Number of used shared buffers", + []string{}, + prometheus.Labels{}, + ) + buffersUnusedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_unused"), + "Number of unused shared buffers", + []string{}, + prometheus.Labels{}, + ) + buffersDirtyDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_dirty"), + "Number of dirty shared buffers", + []string{}, + prometheus.Labels{}, + ) + buffersPinnedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_pinned"), + "Number of pinned shared buffers", + []string{}, + prometheus.Labels{}, + ) + usageCountAvgDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, buffercacheSubsystem, "usagecount_avg"), + "Average usage count of used shared buffers", + []string{}, + prometheus.Labels{}, + ) + + buffercacheQuery = ` + SELECT + buffers_used, + buffers_unused, + buffers_dirty, + buffers_pinned, + usagecount_avg + FROM + pg_buffercache_summary() + ` +) + +func gaugeInt32(m sql.NullInt32, desc *prometheus.Desc, ch chan<- prometheus.Metric) { + mM := 0.0 + if m.Valid { + mM = float64(m.Int32) + } + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, mM) +} + +// Update implements Collector +// It is called by the Prometheus registry when collecting metrics. +func (c BuffercacheCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + // pg_buffercache_summary is only in v16, and we don't need support for earlier currently. + if !instance.version.GE(semver.MustParse("16.0.0")) { + return nil + } + db := instance.getDB() + rows, err := db.QueryContext(ctx, buffercacheQuery) + if err != nil { + return err + } + defer rows.Close() + + var used, unused, dirty, pinned sql.NullInt32 + var usagecountAvg sql.NullFloat64 + + for rows.Next() { + if err := rows.Scan( + &used, + &unused, + &dirty, + &pinned, + &usagecountAvg, + ); err != nil { + return err + } + + usagecountAvgMetric := 0.0 + if usagecountAvg.Valid { + usagecountAvgMetric = usagecountAvg.Float64 + } + ch <- prometheus.MustNewConstMetric( + usageCountAvgDesc, + prometheus.GaugeValue, + usagecountAvgMetric) + gaugeInt32(used, buffersUsedDesc, ch) + gaugeInt32(unused, buffersUnusedDesc, ch) + gaugeInt32(dirty, buffersDirtyDesc, ch) + gaugeInt32(pinned, buffersPinnedDesc, ch) + } + + return rows.Err() +} diff --git a/collector/pg_buffercache_test.go b/collector/pg_buffercache_test.go new file mode 100644 index 000000000..89200ecd3 --- /dev/null +++ b/collector/pg_buffercache_test.go @@ -0,0 +1,73 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/blang/semver/v4" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestBuffercacheCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("16.0.0")} + + columns := []string{ + "buffers_used", + "buffers_unused", + "buffers_dirty", + "buffers_pinned", + "usagecount_avg"} + + rows := sqlmock.NewRows(columns).AddRow(123, 456, 789, 234, 56.6778) + + mock.ExpectQuery(sanitizeQuery(buffercacheQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := BuffercacheCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{}, metricType: dto.MetricType_GAUGE, value: 56.6778}, + {labels: labelMap{}, metricType: dto.MetricType_GAUGE, value: 123}, + {labels: labelMap{}, metricType: dto.MetricType_GAUGE, value: 456}, + {labels: labelMap{}, metricType: dto.MetricType_GAUGE, value: 789}, + {labels: labelMap{}, metricType: dto.MetricType_GAUGE, value: 234}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 6014965d848d534bb490672e95daac505ffa14a7 Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Wed, 18 Jun 2025 16:01:35 +0000 Subject: [PATCH 2/6] Update collector/pg_buffercache.go Co-authored-by: Ben Kochie Signed-off-by: Peter Nuttall --- collector/pg_buffercache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_buffercache.go b/collector/pg_buffercache.go index 8de1329aa..f0ad509a5 100644 --- a/collector/pg_buffercache.go +++ b/collector/pg_buffercache.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at From 64f002fe8f59d8df0004301405d716d78a9e95b5 Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Wed, 18 Jun 2025 16:17:27 +0000 Subject: [PATCH 3/6] Rename `s/pg_buffercache/pg_buffercache_summary/`. Signed-off-by: Peter Nuttall --- ...ffercache.go => pg_buffercache_summary.go} | 24 +++++++++---------- ...test.go => pg_buffercache_summary_test.go} | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) rename collector/{pg_buffercache.go => pg_buffercache_summary.go} (75%) rename collector/{pg_buffercache_test.go => pg_buffercache_summary_test.go} (96%) diff --git a/collector/pg_buffercache.go b/collector/pg_buffercache_summary.go similarity index 75% rename from collector/pg_buffercache.go rename to collector/pg_buffercache_summary.go index 8de1329aa..8e33fdef2 100644 --- a/collector/pg_buffercache.go +++ b/collector/pg_buffercache_summary.go @@ -22,56 +22,56 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const buffercacheSubsystem = "buffercache" +const buffercacheSummarySubsystem = "buffercache_summary" func init() { - registerCollector(buffercacheSubsystem, defaultDisabled, NewBuffercacheCollector) + registerCollector(buffercacheSummarySubsystem, defaultDisabled, NewBuffercacheSummaryCollector) } -// BuffercacheCollector collects stats from pg_buffercache: https://www.postgresql.org/docs/current/pgbuffercache.html. +// BuffercacheSummaryCollector collects stats from pg_buffercache: https://www.postgresql.org/docs/current/pgbuffercache.html. // // It depends on the extension being loaded with // // create extension pg_buffercache; // // It does not take locks, see the PG docs above. -type BuffercacheCollector struct { +type BuffercacheSummaryCollector struct { log *slog.Logger } -func NewBuffercacheCollector(config collectorConfig) (Collector, error) { - return &BuffercacheCollector{ +func NewBuffercacheSummaryCollector(config collectorConfig) (Collector, error) { + return &BuffercacheSummaryCollector{ log: config.logger, }, nil } var ( buffersUsedDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_used"), + prometheus.BuildFQName(namespace, buffercacheSummarySubsystem, "buffers_used"), "Number of used shared buffers", []string{}, prometheus.Labels{}, ) buffersUnusedDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_unused"), + prometheus.BuildFQName(namespace, buffercacheSummarySubsystem, "buffers_unused"), "Number of unused shared buffers", []string{}, prometheus.Labels{}, ) buffersDirtyDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_dirty"), + prometheus.BuildFQName(namespace, buffercacheSummarySubsystem, "buffers_dirty"), "Number of dirty shared buffers", []string{}, prometheus.Labels{}, ) buffersPinnedDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, buffercacheSubsystem, "buffers_pinned"), + prometheus.BuildFQName(namespace, buffercacheSummarySubsystem, "buffers_pinned"), "Number of pinned shared buffers", []string{}, prometheus.Labels{}, ) usageCountAvgDesc = prometheus.NewDesc( - prometheus.BuildFQName(namespace, buffercacheSubsystem, "usagecount_avg"), + prometheus.BuildFQName(namespace, buffercacheSummarySubsystem, "usagecount_avg"), "Average usage count of used shared buffers", []string{}, prometheus.Labels{}, @@ -99,7 +99,7 @@ func gaugeInt32(m sql.NullInt32, desc *prometheus.Desc, ch chan<- prometheus.Met // Update implements Collector // It is called by the Prometheus registry when collecting metrics. -func (c BuffercacheCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { +func (c BuffercacheSummaryCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { // pg_buffercache_summary is only in v16, and we don't need support for earlier currently. if !instance.version.GE(semver.MustParse("16.0.0")) { return nil diff --git a/collector/pg_buffercache_test.go b/collector/pg_buffercache_summary_test.go similarity index 96% rename from collector/pg_buffercache_test.go rename to collector/pg_buffercache_summary_test.go index 89200ecd3..2d83801ef 100644 --- a/collector/pg_buffercache_test.go +++ b/collector/pg_buffercache_summary_test.go @@ -23,7 +23,7 @@ import ( "github.com/smartystreets/goconvey/convey" ) -func TestBuffercacheCollector(t *testing.T) { +func TestBuffercacheSummaryCollector(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) @@ -46,7 +46,7 @@ func TestBuffercacheCollector(t *testing.T) { ch := make(chan prometheus.Metric) go func() { defer close(ch) - c := BuffercacheCollector{} + c := BuffercacheSummaryCollector{} if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) From aa004f69302c6992c77401030c678bd1fba44ceb Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Wed, 18 Jun 2025 18:14:41 +0000 Subject: [PATCH 4/6] Update collector/pg_buffercache_summary.go Co-authored-by: Ben Kochie Signed-off-by: Peter Nuttall --- collector/pg_buffercache_summary.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_buffercache_summary.go b/collector/pg_buffercache_summary.go index cdd16d61e..3dc67ad88 100644 --- a/collector/pg_buffercache_summary.go +++ b/collector/pg_buffercache_summary.go @@ -89,7 +89,7 @@ var ( ` ) -func gaugeInt32(m sql.NullInt32, desc *prometheus.Desc, ch chan<- prometheus.Metric) { +func gaugeInt32(ch chan<- prometheus.Metric, desc *prometheus.Desc, m sql.NullInt32) { mM := 0.0 if m.Valid { mM = float64(m.Int32) From 6e53796dfd5b4aa8c6f39dff1c1c7e989865d0f2 Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Wed, 18 Jun 2025 18:18:05 +0000 Subject: [PATCH 5/6] Move gaugeInt32 into collector.go. Signed-off-by: Peter Nuttall --- collector/collector.go | 9 +++++++++ collector/pg_buffercache_summary.go | 16 ++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index 298bc36ee..84136e287 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "errors" "fmt" "log/slog" @@ -228,3 +229,11 @@ var ErrNoData = errors.New("collector returned no data") func IsNoDataError(err error) bool { return err == ErrNoData } + +func Int32(m sql.NullInt32) float64 { + mM := 0.0 + if m.Valid { + mM = float64(m.Int32) + } + return mM +} diff --git a/collector/pg_buffercache_summary.go b/collector/pg_buffercache_summary.go index 3dc67ad88..8b0e0f007 100644 --- a/collector/pg_buffercache_summary.go +++ b/collector/pg_buffercache_summary.go @@ -89,14 +89,6 @@ var ( ` ) -func gaugeInt32(ch chan<- prometheus.Metric, desc *prometheus.Desc, m sql.NullInt32) { - mM := 0.0 - if m.Valid { - mM = float64(m.Int32) - } - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, mM) -} - // Update implements Collector // It is called by the Prometheus registry when collecting metrics. func (c BuffercacheSummaryCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { @@ -133,10 +125,10 @@ func (c BuffercacheSummaryCollector) Update(ctx context.Context, instance *insta usageCountAvgDesc, prometheus.GaugeValue, usagecountAvgMetric) - gaugeInt32(used, buffersUsedDesc, ch) - gaugeInt32(unused, buffersUnusedDesc, ch) - gaugeInt32(dirty, buffersDirtyDesc, ch) - gaugeInt32(pinned, buffersPinnedDesc, ch) + ch <- prometheus.MustNewConstMetric(buffersUsedDesc, prometheus.GaugeValue, Int32(used)) + ch <- prometheus.MustNewConstMetric(buffersUnusedDesc, prometheus.GaugeValue, Int32(unused)) + ch <- prometheus.MustNewConstMetric(buffersDirtyDesc, prometheus.GaugeValue, Int32(dirty)) + ch <- prometheus.MustNewConstMetric(buffersPinnedDesc, prometheus.GaugeValue, Int32(pinned)) } return rows.Err() From 62bcb1289cfcbe2490af58de765d3d442f086c59 Mon Sep 17 00:00:00 2001 From: Peter Nuttall Date: Wed, 18 Jun 2025 18:19:25 +0000 Subject: [PATCH 6/6] copyright Signed-off-by: Peter Nuttall --- collector/pg_buffercache_summary_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_buffercache_summary_test.go b/collector/pg_buffercache_summary_test.go index 2d83801ef..86d91751f 100644 --- a/collector/pg_buffercache_summary_test.go +++ b/collector/pg_buffercache_summary_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at