diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4ad620e..b3e64fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,9 @@ +0.9.13 - 2019.10.01 +################### + +* Add support for PAY_PER_REQUEST billing mode +* Bump minimum version of boto3 to 1.9.54 + 0.9.12 - 2019.09.30 ################### diff --git a/dynamorm/exceptions.py b/dynamorm/exceptions.py index 4cb840e..c551ce0 100644 --- a/dynamorm/exceptions.py +++ b/dynamorm/exceptions.py @@ -43,6 +43,10 @@ class MissingTableAttribute(DynamoTableException): """A required attribute is missing""" +class InvalidTableAttribute(DynamoTableException): + """An attribute has an invalid value""" + + class InvalidSchemaField(DynamoTableException): """A field provided does not exist in the schema""" diff --git a/dynamorm/table.py b/dynamorm/table.py index de3cd06..d3271df 100644 --- a/dynamorm/table.py +++ b/dynamorm/table.py @@ -4,25 +4,27 @@ The attributes you define on your inner ``Table`` class map to underlying boto data structures. This mapping is expressed through the following data model: -========= ======== ==== =========== -Attribute Required Type Description -========= ======== ==== =========== -name True str The name of the table, as stored in Dynamo. +========= ======== ==== =========== +Attribute Required Type Description +========= ======== ==== =========== +name True str The name of the table, as stored in Dynamo. -hash_key True str The name of the field to use as the hash key. +hash_key True str The name of the field to use as the hash key. It must exist in the schema. -range_key False str The name of the field to use as the range_key, if one is used. +range_key False str The name of the field to use as the range_key, if one is used. It must exist in the schema. -read True int The provisioned read throughput. +read Cond int The provisioned read throughput. Required for 'PROVISIONED' billing_mode (default). -write True int The provisioned write throughput. +write Cond int The provisioned write throughput. Required for 'PROVISIONED' billing_mode (default). -stream False str The stream view type, either None or one of: - 'NEW_IMAGE'|'OLD_IMAGE'|'NEW_AND_OLD_IMAGES'|'KEYS_ONLY' +billing_mode True str The billing mode. One of: 'PROVISIONED'|'PAY_PER_REQUEST' -========= ======== ==== =========== +stream False str The stream view type, either None or one of: + 'NEW_IMAGE'|'OLD_IMAGE'|'NEW_AND_OLD_IMAGES'|'KEYS_ONLY' + +========= ======== ==== =========== Indexes @@ -52,6 +54,7 @@ from boto3.dynamodb.conditions import Key, Attr from dynamorm.exceptions import ( + InvalidTableAttribute, MissingTableAttribute, TableNotActive, InvalidSchemaField, @@ -72,6 +75,7 @@ class DynamoCommon3(object): range_key = None read = None write = None + billing_mode = "PROVISIONED" def __init__(self): for attr in self.REQUIRED_ATTRS: @@ -167,7 +171,8 @@ class DynamoGlobalIndex3(DynamoIndex3): @property def index_args(self): args = super(DynamoGlobalIndex3, self).index_args - args["ProvisionedThroughput"] = self.provisioned_throughput + if self.table.billing_mode == "PROVISIONED": + args["ProvisionedThroughput"] = self.provisioned_throughput return args @@ -188,6 +193,17 @@ def __init__(self, schema, indexes=None): super(DynamoTable3, self).__init__() + if self.billing_mode not in ("PROVISIONED", "PAY_PER_REQUEST"): + raise InvalidTableAttribute( + "valid values for billing_mode are: PROVISIONED|PAY_PER_REQUEST" + ) + + if self.billing_mode == "PROVISIONED" and (not self.read or not self.write): + raise MissingTableAttribute( + "The read/write attributes are required to create " + "a table when billing_mode is 'PROVISIONED'" + ) + self.indexes = {} if indexes: for name, klass in six.iteritems(indexes): @@ -344,23 +360,21 @@ def create_table(self, wait=True): :param bool wait: If set to True, the default, this call will block until the table is created """ - if not self.read or not self.write: - raise MissingTableAttribute( - "The read/write attributes are required to create a table" - ) - - index_args = collections.defaultdict(list) + extra_args = collections.defaultdict(list) for index in six.itervalues(self.indexes): - index_args[index.ARG_KEY].append(index.index_args) + extra_args[index.ARG_KEY].append(index.index_args) + + if self.billing_mode == "PROVISIONED": + extra_args["ProvisionedThroughput"] = self.provisioned_throughput log.info("Creating table %s", self.name) table = self.resource.create_table( TableName=self.name, KeySchema=self.key_schema, AttributeDefinitions=self.attribute_definitions, - ProvisionedThroughput=self.provisioned_throughput, StreamSpecification=self.stream_specification, - **index_args + BillingMode=self.billing_mode, + **extra_args ) if wait: log.info("Waiting for table creation...") @@ -446,8 +460,21 @@ def do_update(**kwargs): wait_for_active() + billing_args = {} + + # check if we're going to change our billing mode + current_billing_mode = table.billing_mode_summary["BillingMode"] + if self.billing_mode != current_billing_mode: + log.info( + "Updating billing mode on table %s (%s -> %s)", + self.name, + current_billing_mode, + self.billing_mode, + ) + billing_args["BillingMode"] = self.billing_mode + # check if we're going to change our capacity - if (self.read and self.write) and ( + if (self.billing_mode == "PROVISIONED" and self.read and self.write) and ( self.read != table.provisioned_throughput["ReadCapacityUnits"] or self.write != table.provisioned_throughput["WriteCapacityUnits"] ): @@ -462,7 +489,10 @@ def do_update(**kwargs): ), self.provisioned_throughput, ) - do_update(ProvisionedThroughput=self.provisioned_throughput) + billing_args["ProvisionedThroughput"] = self.provisioned_throughput + + if billing_args: + do_update(**billing_args) return self.update_table() # check if we're going to modify the stream @@ -503,7 +533,11 @@ def do_update(**kwargs): for index in six.itervalues(self.indexes): if index.name in existing_indexes: current_capacity = existing_indexes[index.name]["ProvisionedThroughput"] - if (index.read and index.write) and ( + update_args = {} + + if ( + index.billing_mode == "PROVISIONED" and index.read and index.write + ) and ( index.read != current_capacity["ReadCapacityUnits"] or index.write != current_capacity["WriteCapacityUnits"] ): @@ -519,7 +553,7 @@ def do_update(**kwargs): GlobalSecondaryIndexUpdates=[ { "Update": { - "IndexName": index["IndexName"], + "IndexName": index.name, "ProvisionedThroughput": index.provisioned_throughput, } } diff --git a/setup.py b/setup.py index 284f7fa..0c9c72d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dynamorm", - version="0.9.12", + version="0.9.13", description="DynamORM is a Python object & relation mapping library for Amazon's DynamoDB service.", long_description=long_description, author="Evan Borgstrom", @@ -13,7 +13,7 @@ url="https://github.com/NerdWalletOSS/DynamORM", license="Apache License Version 2.0", python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", - install_requires=["blinker>=1.4,<2.0", "boto3>=1.3,<2.0", "six"], + install_requires=["blinker>=1.4,<2.0", "boto3>=1.9.54,<2.0", "six"], extras_require={ "marshmallow": ["marshmallow>=2.15.1,<4"], "schematics": ["schematics>=2.1.0,<3"], diff --git a/tests/test_model.py b/tests/test_model.py index 3ee00c5..e7a7124 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -7,6 +7,7 @@ DynaModelException, HashKeyExists, InvalidSchemaField, + InvalidTableAttribute, MissingTableAttribute, ValidationError, ) @@ -79,10 +80,31 @@ class Child(Parent): def test_table_validation(): """Defining a model with missing table attributes should raise exceptions""" with pytest.raises(MissingTableAttribute): + # Missing hash_key + class Model(DynaModel): + class Table: + name = "table" + + class Schema: + foo = String(required=True) + + with pytest.raises(MissingTableAttribute): + # Missing read/write + class Model(DynaModel): + class Table: + name = "table" + hash_key = "foo" + class Schema: + foo = String(required=True) + + with pytest.raises(InvalidTableAttribute): + # Invalid billing mode class Model(DynaModel): class Table: name = "table" + hash_key = "foo" + billing_mode = "FOO" class Schema: foo = String(required=True)