diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0f37fe9e0..60fd7ecacfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * [FEATURE] Ingester: Add active series to all_user_stats page. #4972 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005 +* [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008 ## 1.14.0 in progress diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index f6f9bd2d467..af2edea1cbf 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -882,4 +882,9 @@ blocks_storage: # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] + + # True to enable snapshotting of in-memory TSDB data on disk when shutting + # down. + # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown + [memory_snapshot_on_shutdown: | default = false] ``` diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 2c6a66d26ee..a789a9a5607 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -946,4 +946,9 @@ blocks_storage: # will be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] + + # True to enable snapshotting of in-memory TSDB data on disk when shutting + # down. + # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown + [memory_snapshot_on_shutdown: | default = false] ``` diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 50d1ae85f1b..926edb10424 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3725,6 +3725,11 @@ tsdb: # be stored. 0 or less means disabled. # CLI flag: -blocks-storage.tsdb.max-exemplars [max_exemplars: | default = 0] + + # True to enable snapshotting of in-memory TSDB data on disk when shutting + # down. + # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown + [memory_snapshot_on_shutdown: | default = false] ``` ### `compactor_config` diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index 2dc2db88bf3..004f8aabb25 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -98,4 +98,6 @@ Currently experimental features are: - Enabled via `-compactor.sharding-enabled=true`, `-compactor.sharding-strategy=shuffle-sharding`, and `-compactor.tenant-shard-size` set to a value larger than 0. - Vertical sharding at query frontend for range/instant queries - `-frontend.query-vertical-shard-size` (int) CLI flag - - `query_vertical_shard_size` (int) field in runtime config file \ No newline at end of file + - `query_vertical_shard_size` (int) field in runtime config file +- Snapshotting of in-memory TSDB on disk during shutdown + - `-blocks-storage.tsdb.memory-snapshot-on-shutdown` (boolean) CLI flag \ No newline at end of file diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 8593c89952a..9f776bd93bc 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1833,20 +1833,21 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { } // Create a new user database db, err := tsdb.Open(udir, userLogger, tsdbPromReg, &tsdb.Options{ - RetentionDuration: i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(), - MinBlockDuration: blockRanges[0], - MaxBlockDuration: blockRanges[len(blockRanges)-1], - NoLockfile: true, - StripeSize: i.cfg.BlocksStorageConfig.TSDB.StripeSize, - HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize, - WALCompression: i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled, - WALSegmentSize: i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes, - SeriesLifecycleCallback: userDB, - BlocksToDelete: userDB.blocksToDelete, - EnableExemplarStorage: enableExemplars, - IsolationDisabled: true, - MaxExemplars: int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars), - HeadChunksWriteQueueSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize, + RetentionDuration: i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(), + MinBlockDuration: blockRanges[0], + MaxBlockDuration: blockRanges[len(blockRanges)-1], + NoLockfile: true, + StripeSize: i.cfg.BlocksStorageConfig.TSDB.StripeSize, + HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize, + WALCompression: i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled, + WALSegmentSize: i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes, + SeriesLifecycleCallback: userDB, + BlocksToDelete: userDB.blocksToDelete, + EnableExemplarStorage: enableExemplars, + IsolationDisabled: true, + MaxExemplars: int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars), + HeadChunksWriteQueueSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize, + EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, }, nil) if err != nil { return nil, errors.Wrapf(err, "failed to open TSDB: %s", udir) diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index ce729210389..c8009066364 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -151,6 +151,9 @@ type TSDBConfig struct { // Positive value enables experiemental support for exemplars. 0 or less to disable. MaxExemplars int `yaml:"max_exemplars"` + + // Enable snapshotting of in-memory TSDB data on disk when shutting down. + MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"` } // RegisterFlags registers the TSDBConfig flags. @@ -176,6 +179,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value should be equal or higher than -querier.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.") f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.") f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.") + f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") } // Validate the config.