From 9e3603b3f5c9fb2172807e2a4ec3e9a54fad3384 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Mon, 28 Jun 2021 16:19:59 -0400 Subject: [PATCH 1/7] add timeseries options for collection creation --- src/db/options.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/db/options.rs b/src/db/options.rs index fec33591e..0df235731 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -83,6 +83,9 @@ pub struct CreateCollectionOptions { /// The default configuration for indexes created on this collection, including the _id index. pub index_option_defaults: Option, + + /// Specifies options for creating a timeseries collection. + pub timeseries: Option, } /// Specifies how strictly the database should apply validation rules to existing documents during @@ -123,6 +126,25 @@ pub struct IndexOptionDefaults { pub storage_engine: Document, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] + +/// Specifies options for creating a timeseries collection. +pub struct TimeseriesOptions { + /// Name of the top-level field to be used for time. Inserted documents must have this field, + /// and the field must be of the BSON UTC datetime type. + pub time_field: String, + + /// Name of the top-level field describing the series. This field is used to group related data + /// and may be of any BSON type, except for array. This name may not be the same as the + /// timeField or _id. + pub meta_field: Option, + + /// Number indicating after how many seconds old time-series data should be deleted. + pub expire_after_seconds: Option, +} + /// Specifies the options to a [`Database::drop`](../struct.Database.html#method.drop) operation. #[derive(Debug, Default, TypedBuilder, Serialize)] #[serde(rename_all = "camelCase")] From b9d35f1882852dfd1653e841bad94bbf768be60c Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 29 Jun 2021 10:58:30 -0400 Subject: [PATCH 2/7] pull in collection-management tests; shift expire_after_seconds up a level --- src/db/options.rs | 6 +- src/test/spec/collection_management.rs | 12 + .../json/collection-management/README.rst | 7 + .../timeseries-collection.json | 255 ++++++++++++++++++ .../timeseries-collection.yml | 129 +++++++++ src/test/spec/mod.rs | 1 + 6 files changed, 407 insertions(+), 3 deletions(-) create mode 100644 src/test/spec/collection_management.rs create mode 100644 src/test/spec/json/collection-management/README.rst create mode 100644 src/test/spec/json/collection-management/timeseries-collection.json create mode 100644 src/test/spec/json/collection-management/timeseries-collection.yml diff --git a/src/db/options.rs b/src/db/options.rs index 0df235731..27fa47d2a 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -86,6 +86,9 @@ pub struct CreateCollectionOptions { /// Specifies options for creating a timeseries collection. pub timeseries: Option, + + /// Number indicating after how many seconds old time-series data should be deleted. + pub expire_after_seconds: Option, } /// Specifies how strictly the database should apply validation rules to existing documents during @@ -140,9 +143,6 @@ pub struct TimeseriesOptions { /// and may be of any BSON type, except for array. This name may not be the same as the /// timeField or _id. pub meta_field: Option, - - /// Number indicating after how many seconds old time-series data should be deleted. - pub expire_after_seconds: Option, } /// Specifies the options to a [`Database::drop`](../struct.Database.html#method.drop) operation. diff --git a/src/test/spec/collection_management.rs b/src/test/spec/collection_management.rs new file mode 100644 index 000000000..da859f730 --- /dev/null +++ b/src/test/spec/collection_management.rs @@ -0,0 +1,12 @@ +use crate::{ + test::{run_spec_test, LOCK}, +}; + +use super::run_unified_format_test; + +#[cfg_attr(feature = "tokio-runtime", tokio::test)] +#[cfg_attr(feature = "async-std-runtime", async_std::test)] +async fn run() { + let _guard = LOCK.run_exclusively().await; + run_spec_test(&["collection-management"], run_unified_format_test).await; +} diff --git a/src/test/spec/json/collection-management/README.rst b/src/test/spec/json/collection-management/README.rst new file mode 100644 index 000000000..940161b9a --- /dev/null +++ b/src/test/spec/json/collection-management/README.rst @@ -0,0 +1,7 @@ +=========================== +Collection Management Tests +=========================== + +This directory contains tests for collection management. They are implemented +in the `Unified Test Format <../../unified-test-format/unified-test-format.rst>`__ +and require schema version 1.0. diff --git a/src/test/spec/json/collection-management/timeseries-collection.json b/src/test/spec/json/collection-management/timeseries-collection.json new file mode 100644 index 000000000..b5638fd36 --- /dev/null +++ b/src/test/spec/json/collection-management/timeseries-collection.json @@ -0,0 +1,255 @@ +{ + "description": "timeseries-collection", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "5.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "ts-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "ts-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "createCollection with all options", + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "createCollection", + "object": "database0", + "arguments": { + "collection": "test", + "expireAfterSeconds": 604800, + "timeseries": { + "timeField": "time", + "metaField": "meta", + "granularity": "minutes" + } + } + }, + { + "name": "assertCollectionExists", + "object": "testRunner", + "arguments": { + "databaseName": "ts-tests", + "collectionName": "test" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test" + }, + "databaseName": "ts-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "create": "test", + "expireAfterSeconds": 604800, + "timeseries": { + "timeField": "time", + "metaField": "meta", + "granularity": "minutes" + } + }, + "databaseName": "ts-tests" + } + } + ] + } + ] + }, + { + "description": "insertMany with duplicate ids", + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "createCollection", + "object": "database0", + "arguments": { + "collection": "test", + "expireAfterSeconds": 604800, + "timeseries": { + "timeField": "time", + "metaField": "meta", + "granularity": "minutes" + } + } + }, + { + "name": "assertCollectionExists", + "object": "testRunner", + "arguments": { + "databaseName": "ts-tests", + "collectionName": "test" + } + }, + { + "name": "insertMany", + "object": "collection0", + "arguments": { + "documents": [ + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630482" + } + } + }, + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630483" + } + } + } + ] + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {}, + "sort": { + "time": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630482" + } + } + }, + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630483" + } + } + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test" + }, + "databaseName": "ts-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "create": "test", + "expireAfterSeconds": 604800, + "timeseries": { + "timeField": "time", + "metaField": "meta", + "granularity": "minutes" + } + }, + "databaseName": "ts-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630482" + } + } + }, + { + "_id": 1, + "time": { + "$date": { + "$numberLong": "1552949630483" + } + } + } + ] + } + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": {}, + "sort": { + "time": 1 + } + }, + "databaseName": "ts-tests" + } + } + ] + } + ] + } + ] +} diff --git a/src/test/spec/json/collection-management/timeseries-collection.yml b/src/test/spec/json/collection-management/timeseries-collection.yml new file mode 100644 index 000000000..cbc09b34c --- /dev/null +++ b/src/test/spec/json/collection-management/timeseries-collection.yml @@ -0,0 +1,129 @@ +description: "timeseries-collection" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "5.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name ts-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + +tests: + - description: "createCollection with all options" + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: createCollection + object: *database0 + arguments: + collection: *collection0Name + # expireAfterSeconds should be an int64 (as it is stored on the server). + expireAfterSeconds: 604800 + timeseries: ×eries0 + timeField: "time" + metaField: "meta" + granularity: "minutes" + - name: assertCollectionExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + databaseName: *database0Name + - commandStartedEvent: + command: + create: *collection0Name + expireAfterSeconds: 604800 + timeseries: *timeseries0 + databaseName: *database0Name + + # Unlike regular collections, time-series collections allow duplicate ids. + - description: "insertMany with duplicate ids" + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: createCollection + object: *database0 + arguments: + collection: *collection0Name + # expireAfterSeconds should be an int64 (as it is stored on the server). + expireAfterSeconds: 604800 + timeseries: *timeseries0 + - name: assertCollectionExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + - name: insertMany + object: *collection0 + arguments: + documents: &docs + - { + _id: 1, + time: { + $date: { + $numberLong: "1552949630482" + } + } + } + - { + _id: 1, + time: { + $date: { + $numberLong: "1552949630483" + } + } + } + - name: find + object: *collection0 + arguments: + filter: {} + sort: { time: 1 } + expectResult: *docs + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + databaseName: *database0Name + - commandStartedEvent: + command: + create: *collection0Name + expireAfterSeconds: 604800 + timeseries: *timeseries0 + databaseName: *database0Name + - commandStartedEvent: + command: + insert: *collection0Name + documents: *docs + - commandStartedEvent: + command: + find: *collection0Name + filter: {} + sort: { time: 1 } + databaseName: *database0Name diff --git a/src/test/spec/mod.rs b/src/test/spec/mod.rs index 434594139..4c53f5f1f 100644 --- a/src/test/spec/mod.rs +++ b/src/test/spec/mod.rs @@ -1,5 +1,6 @@ #[cfg(not(feature = "sync"))] mod auth; +mod collection_management; mod command_monitoring; mod connection_stepdown; mod crud_unified; From 6733395ba48a5b5b268774f7b99f700d8837e561 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 29 Jun 2021 15:11:00 -0400 Subject: [PATCH 3/7] properly [de]serialize expire_after_seconds, and expose granularity --- src/bson_util/mod.rs | 1 - src/db/options.rs | 24 +++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/bson_util/mod.rs b/src/bson_util/mod.rs index f2d329303..2f3498567 100644 --- a/src/bson_util/mod.rs +++ b/src/bson_util/mod.rs @@ -82,7 +82,6 @@ pub(crate) fn serialize_duration_as_int_millis( } } -#[cfg(test)] pub(crate) fn serialize_duration_option_as_int_secs( val: &Option, serializer: S, diff --git a/src/db/options.rs b/src/db/options.rs index 27fa47d2a..b50ebb03e 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; use typed_builder::TypedBuilder; @@ -88,7 +90,9 @@ pub struct CreateCollectionOptions { pub timeseries: Option, /// Number indicating after how many seconds old time-series data should be deleted. - pub expire_after_seconds: Option, + #[serde(deserialize_with = "bson_util::deserialize_duration_from_u64_seconds", + serialize_with = "bson_util::serialize_duration_option_as_int_secs")] + pub expire_after_seconds: Option, } /// Specifies how strictly the database should apply validation rules to existing documents during @@ -143,6 +147,24 @@ pub struct TimeseriesOptions { /// and may be of any BSON type, except for array. This name may not be the same as the /// timeField or _id. pub meta_field: Option, + + /// The units you'd use to describe the expected interval between subsequent measurements for a + /// time-series. Defaults to `TimeseriesGranularity::Seconds` if unset. + pub granularity: Option, +} + +/// The units you'd use to describe the expected interval between subsequent measurements for a +/// time-series. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub enum TimeseriesGranularity { + /// The expected interval between subsequent measurements is in seconds. + Seconds, + /// The expected interval between subsequent measurements is in minutes. + Minutes, + /// The expected interval between subsequent measurements is in hours. + Hours, } /// Specifies the options to a [`Database::drop`](../struct.Database.html#method.drop) operation. From 0a38dfa1838c5e8f963d0fa6fe683611afa685d9 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 29 Jun 2021 16:05:59 -0400 Subject: [PATCH 4/7] serde(default) for expire_after_seconds --- src/db/options.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db/options.rs b/src/db/options.rs index b50ebb03e..efd5abc6d 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -89,8 +89,10 @@ pub struct CreateCollectionOptions { /// Specifies options for creating a timeseries collection. pub timeseries: Option, - /// Number indicating after how many seconds old time-series data should be deleted. - #[serde(deserialize_with = "bson_util::deserialize_duration_from_u64_seconds", + /// Duration indicating after how long old time-series data should be deleted. + #[serde( + default, + deserialize_with = "bson_util::deserialize_duration_from_u64_seconds", serialize_with = "bson_util::serialize_duration_option_as_int_secs")] pub expire_after_seconds: Option, } From cf1c6a20d479f75d694d3f0d8fafc0bd03660a57 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 29 Jun 2021 16:50:15 -0400 Subject: [PATCH 5/7] rustfmt --- src/db/options.rs | 3 ++- src/test/spec/collection_management.rs | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/db/options.rs b/src/db/options.rs index efd5abc6d..6befad0e5 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -93,7 +93,8 @@ pub struct CreateCollectionOptions { #[serde( default, deserialize_with = "bson_util::deserialize_duration_from_u64_seconds", - serialize_with = "bson_util::serialize_duration_option_as_int_secs")] + serialize_with = "bson_util::serialize_duration_option_as_int_secs" + )] pub expire_after_seconds: Option, } diff --git a/src/test/spec/collection_management.rs b/src/test/spec/collection_management.rs index da859f730..2dfc32a51 100644 --- a/src/test/spec/collection_management.rs +++ b/src/test/spec/collection_management.rs @@ -1,6 +1,4 @@ -use crate::{ - test::{run_spec_test, LOCK}, -}; +use crate::test::{run_spec_test, LOCK}; use super::run_unified_format_test; From 2211c9512942244c467e047b7baccf5d0e577930 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Tue, 29 Jun 2021 16:52:31 -0400 Subject: [PATCH 6/7] formatting --- src/db/options.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/db/options.rs b/src/db/options.rs index 6befad0e5..e602891c7 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -136,11 +136,10 @@ pub struct IndexOptionDefaults { pub storage_engine: Document, } +/// Specifies options for creating a timeseries collection. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] #[non_exhaustive] - -/// Specifies options for creating a timeseries collection. pub struct TimeseriesOptions { /// Name of the top-level field to be used for time. Inserted documents must have this field, /// and the field must be of the BSON UTC datetime type. From b76e001bd4d06a89b3c243d904f5298317615e24 Mon Sep 17 00:00:00 2001 From: Abraham Egnor Date: Wed, 30 Jun 2021 12:07:04 -0400 Subject: [PATCH 7/7] Update comment to indicate version support --- src/db/options.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db/options.rs b/src/db/options.rs index e602891c7..570c1e454 100644 --- a/src/db/options.rs +++ b/src/db/options.rs @@ -86,7 +86,8 @@ pub struct CreateCollectionOptions { /// The default configuration for indexes created on this collection, including the _id index. pub index_option_defaults: Option, - /// Specifies options for creating a timeseries collection. + /// Specifies options for creating a timeseries collection. This feature is only available on + /// server versions 5.0 and above. pub timeseries: Option, /// Duration indicating after how long old time-series data should be deleted.