From 5d6eb92a8be9b9d0332e8a0aa0b0a2e8f9092d93 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 17:46:11 +0000 Subject: [PATCH 01/10] Added unit test --- .idea/workspace.xml | 62 +++++++++++++++++++ .../aws/package/compile/events/kafka.test.js | 32 ++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000000..1204338f53 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + { + "associatedIndex": 7 +} + + + + { + "keyToString": { + "RunOnceActivity.ShowReadmeOnStart": "true", + "RunOnceActivity.git.unshallow": "true", + "git-widget-placeholder": "main", + "last_opened_file_path": "/Users/demetrios.loutsios/Code/serverless", + "node.js.detected.package.eslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "nodejs_package_manager_path": "npm", + "settings.editor.selected.configurable": "preferences.lookFeel", + "vue.rearranger.settings.migration": "true" + } +} + + + + + + + + + + + 1740590742588 + + + + + + \ No newline at end of file diff --git a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js index 58b772096d..27a3a1d99b 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js @@ -673,6 +673,38 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => }); }); + it('should correctly compile EventSourceMapping resource properties for ProvisionedPollerConfig', async () => { + const { awsNaming, cfTemplate } = await runServerless({ + fixture: 'function', + configExt: { + functions: { + basic: { + role: { 'Fn::ImportValue': 'MyImportedRole' }, + events: [ + { + kafka: { + topic, + bootstrapServers: ['abc.xyz:9092'], + accessConfigurations: { saslScram256Auth: saslScram256AuthArn }, + provisionedPollerConfig: { minimumPollers: 2, maximumPollers: 10 }, + }, + }, + ], + }, + }, + }, + command: 'package', + }); + + const eventSourceMappingResource = + cfTemplate.Resources[awsNaming.getKafkaEventLogicalId('basic', 'TestingTopic')]; + + expect(eventSourceMappingResource.Properties.ProvisionedPollerConfig).to.deep.equal({ + MinimumPollers: 2, + MaximumPollers: 3 + }); + }); + describe('when no kafka events are defined', () => { it('should not modify the default IAM role', async () => { const { cfTemplate } = await runServerless({ From 50e5d9ea1b20381c62d92898cd8f84b2359ecfbe Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 18:20:20 +0000 Subject: [PATCH 02/10] Implementation and fixed assert --- .../aws/package/compile/events/kafka.js | 24 +++++++++++++++++++ .../aws/package/compile/events/kafka.test.js | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/plugins/aws/package/compile/events/kafka.js b/lib/plugins/aws/package/compile/events/kafka.js index b15c13fcf7..ab3fc348c7 100644 --- a/lib/plugins/aws/package/compile/events/kafka.js +++ b/lib/plugins/aws/package/compile/events/kafka.js @@ -100,6 +100,22 @@ class AwsCompileKafkaEvents { pattern: '[a-zA-Z0-9-/*:_+=.@-]*', }, filterPatterns: { $ref: '#/definitions/filterPatterns' }, + provisionedPollerConfig: { + type: 'object', + minProperties: 2, + properties: { + minimumPollers: { + type: 'number', + minimum: 1, + maximum: 200, + }, + maximumPollers: { + type: 'number', + minimum: 1, + maximum: 2000, + } + } + } }, additionalProperties: false, required: ['accessConfigurations', 'bootstrapServers', 'topic'], @@ -270,6 +286,14 @@ class AwsCompileKafkaEvents { }; } + const provisionedPollerConfig = event.kafka.provisionedPollerConfig; + if (provisionedPollerConfig) { + kafkaResource.Properties.ProvisionedPollerConfig = { + MinimumPollers: provisionedPollerConfig.minimumPollers, + MaximumPollers: provisionedPollerConfig.maximumPollers + }; + } + cfTemplate.Resources[kafkaEventLogicalId] = kafkaResource; }); diff --git a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js index 27a3a1d99b..fd3741fcf1 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js @@ -701,7 +701,7 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => expect(eventSourceMappingResource.Properties.ProvisionedPollerConfig).to.deep.equal({ MinimumPollers: 2, - MaximumPollers: 3 + MaximumPollers: 10 }); }); From f77923dd40bc32080e0b97d4c5ec7f80833fadc9 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 18:20:48 +0000 Subject: [PATCH 03/10] Removed accidental commit --- .idea/workspace.xml | 62 --------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 1204338f53..0000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - { - "associatedIndex": 7 -} - - - - { - "keyToString": { - "RunOnceActivity.ShowReadmeOnStart": "true", - "RunOnceActivity.git.unshallow": "true", - "git-widget-placeholder": "main", - "last_opened_file_path": "/Users/demetrios.loutsios/Code/serverless", - "node.js.detected.package.eslint": "true", - "node.js.selected.package.eslint": "(autodetect)", - "nodejs_package_manager_path": "npm", - "settings.editor.selected.configurable": "preferences.lookFeel", - "vue.rearranger.settings.migration": "true" - } -} - - - - - - - - - - - 1740590742588 - - - - - - \ No newline at end of file From 9b1a8c0ca8dc1ea3d5ecd94ee86b9359b6ff54eb Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:12:36 +0000 Subject: [PATCH 04/10] Updated docs --- docs/events/kafka.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/events/kafka.md b/docs/events/kafka.md index 66d65d2fde..03774c8760 100644 --- a/docs/events/kafka.md +++ b/docs/events/kafka.md @@ -182,3 +182,24 @@ functions: maximumBatchingWindow: 30 startingPosition: LATEST ``` + +## Setting ProvisionedPollerConfig + +You can set the `provisionedPollerConfig` to configure your Kafka event source to be in provisioned mode. A `minimumPollers` must be configured from 1 to 200 and a `maximumPollers` from 1 to 2000. + +```yml +functions: + compute: + handler: handler.compute + events: + - kafka: + accessConfigurations: + saslScram512Auth: arn:aws:secretsmanager:us-east-1:01234567890:secret:MyBrokerSecretName + topic: MySelfManagedKafkaTopic + bootstrapServers: + - abc3.xyz.com:9092 + - abc2.xyz.com:9092 + provisionedPollerConfig: + minimumPollers: 1 + maximumPollers: 10 +``` From a4d33b6590d41d02a34b3b5e836a09ed288ad334 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:19:33 +0000 Subject: [PATCH 05/10] Updated guides --- docs/guides/serverless.yml.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/guides/serverless.yml.md b/docs/guides/serverless.yml.md index e89c6d3ef6..558ea7ecfa 100644 --- a/docs/guides/serverless.yml.md +++ b/docs/guides/serverless.yml.md @@ -1044,6 +1044,10 @@ functions: # Optional, specifies event pattern content filtering filterPatterns: - eventName: INSERT + # Optional, configures provisioned mode, must specify minimumPollers (1-200 range) and minimumPollers (1-2000 range) + provisionedPollerConfig: + minimumPollers: 1 + maximumPollers: 10 ``` ### RabbitMQ From ac11e4afe9b097e5a08f9683f95db1ea94e1f19e Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:22:34 +0000 Subject: [PATCH 06/10] Extended existing test --- .../lib/plugins/aws/package/compile/events/kafka.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js index fd3741fcf1..990f582e32 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js @@ -21,6 +21,7 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => const batchSize = 5000; const maximumBatchingWindow = 20; const filterPatterns = [{ eventName: 'INSERT' }]; + const provisionedPollerConfig = { minimumPollers: 1, maximumPollers: 10 }; describe('when there are kafka events defined', () => { let minimalEventSourceMappingResource; @@ -56,6 +57,7 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => enabled, startingPosition, filterPatterns, + provisionedPollerConfig }, }, ], @@ -136,6 +138,10 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => }, ], }, + ProvisionedPollerConfig: { + MinimumPollers: 1, + MaximumPollers: 10 + } }); }); }); From 644893a62457c77595a79ebe87677ce9fbe220d0 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:36:25 +0000 Subject: [PATCH 07/10] Relaxed validation - as per docs https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-eventsourcemapping-provisionedpollerconfig.html#cfn-lambda-eventsourcemapping-provisionedpollerconfig-maximumpollers --- lib/plugins/aws/package/compile/events/kafka.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plugins/aws/package/compile/events/kafka.js b/lib/plugins/aws/package/compile/events/kafka.js index ab3fc348c7..80429308e7 100644 --- a/lib/plugins/aws/package/compile/events/kafka.js +++ b/lib/plugins/aws/package/compile/events/kafka.js @@ -102,7 +102,7 @@ class AwsCompileKafkaEvents { filterPatterns: { $ref: '#/definitions/filterPatterns' }, provisionedPollerConfig: { type: 'object', - minProperties: 2, + minProperties: 1, properties: { minimumPollers: { type: 'number', From 4be1c0daaaaa653c649664cab78b594da9b72d2d Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:37:50 +0000 Subject: [PATCH 08/10] Updated doc to highlight only one needs to be specified --- docs/guides/serverless.yml.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/serverless.yml.md b/docs/guides/serverless.yml.md index 558ea7ecfa..f05bf2b4ee 100644 --- a/docs/guides/serverless.yml.md +++ b/docs/guides/serverless.yml.md @@ -1044,7 +1044,7 @@ functions: # Optional, specifies event pattern content filtering filterPatterns: - eventName: INSERT - # Optional, configures provisioned mode, must specify minimumPollers (1-200 range) and minimumPollers (1-2000 range) + # Optional, configures provisioned mode, must specify minimumPollers (1-200 range) or minimumPollers (1-2000 range) provisionedPollerConfig: minimumPollers: 1 maximumPollers: 10 From 11860d78bf16979eb496849ad87a587251db18a8 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Wed, 26 Feb 2025 20:40:29 +0000 Subject: [PATCH 09/10] rewored for clarity --- docs/guides/serverless.yml.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/serverless.yml.md b/docs/guides/serverless.yml.md index f05bf2b4ee..d9c7153d56 100644 --- a/docs/guides/serverless.yml.md +++ b/docs/guides/serverless.yml.md @@ -1044,7 +1044,7 @@ functions: # Optional, specifies event pattern content filtering filterPatterns: - eventName: INSERT - # Optional, configures provisioned mode, must specify minimumPollers (1-200 range) or minimumPollers (1-2000 range) + # Optional, configures provisioned mode, must specify minimumPollers (1-200 range), minimumPollers (1-2000 range). Both or one can be specified. provisionedPollerConfig: minimumPollers: 1 maximumPollers: 10 From 090842a6bc11724c5733b025e0542cf9d169c7e6 Mon Sep 17 00:00:00 2001 From: demetrios-loutsios Date: Thu, 27 Feb 2025 15:42:25 +0000 Subject: [PATCH 10/10] make prettier --- lib/plugins/aws/package/compile/events/kafka.js | 8 ++++---- .../plugins/aws/package/compile/events/kafka.test.js | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/plugins/aws/package/compile/events/kafka.js b/lib/plugins/aws/package/compile/events/kafka.js index 80429308e7..cf43962095 100644 --- a/lib/plugins/aws/package/compile/events/kafka.js +++ b/lib/plugins/aws/package/compile/events/kafka.js @@ -113,9 +113,9 @@ class AwsCompileKafkaEvents { type: 'number', minimum: 1, maximum: 2000, - } - } - } + }, + }, + }, }, additionalProperties: false, required: ['accessConfigurations', 'bootstrapServers', 'topic'], @@ -290,7 +290,7 @@ class AwsCompileKafkaEvents { if (provisionedPollerConfig) { kafkaResource.Properties.ProvisionedPollerConfig = { MinimumPollers: provisionedPollerConfig.minimumPollers, - MaximumPollers: provisionedPollerConfig.maximumPollers + MaximumPollers: provisionedPollerConfig.maximumPollers, }; } diff --git a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js index 990f582e32..d74b459cc1 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/kafka.test.js @@ -57,7 +57,7 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => enabled, startingPosition, filterPatterns, - provisionedPollerConfig + provisionedPollerConfig, }, }, ], @@ -140,8 +140,8 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => }, ProvisionedPollerConfig: { MinimumPollers: 1, - MaximumPollers: 10 - } + MaximumPollers: 10, + }, }); }); }); @@ -706,8 +706,8 @@ describe('test/unit/lib/plugins/aws/package/compile/events/kafka.test.js', () => cfTemplate.Resources[awsNaming.getKafkaEventLogicalId('basic', 'TestingTopic')]; expect(eventSourceMappingResource.Properties.ProvisionedPollerConfig).to.deep.equal({ - MinimumPollers: 2, - MaximumPollers: 10 + MinimumPollers: 2, + MaximumPollers: 10, }); });