From 10f2f7cd854295b72d3a4ac9495e7d06113e1615 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 13:07:54 +0100 Subject: [PATCH 01/11] Allow for region and service variables to be set --- lib/resty/aws.lua | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 03c7acb..9e53f81 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -106,13 +106,15 @@ local function get_service_and_region(host) return nil, nil end -function _M.aws_set_headers(access_key, secret_key, host, uri) +function _M.aws_set_headers(access_key, secret_key, host, uri, region, service) local creds = { access_key = access_key, secret_key = secret_key } local timestamp = tonumber(ngx.time()) - local service, region = get_service_and_region(host) + if region == nil or service == nil then + service, region = get_service_and_region(host) + end local auth = get_authorization(creds, timestamp, region, service, host, uri) ngx.req.set_header('Authorization', auth) @@ -120,8 +122,8 @@ function _M.aws_set_headers(access_key, secret_key, host, uri) ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp)) end -function _M.s3_set_headers(access_key, secret_key, host, uri) - _M.aws_set_headers(access_key, secret_key, host, uri) +function _M.s3_set_headers(access_key, secret_key, host, uri, region, service) + _M.aws_set_headers(access_key, secret_key, host, uri, region service) ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) end From ea24a9f477b0bb0179185b221f7520cb1d6872a0 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 13:08:26 +0100 Subject: [PATCH 02/11] fix case condition warnings --- lib/resty/aws.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 9e53f81..b9ba6d9 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -4,7 +4,7 @@ local resty_hmac = require('resty.hmac') local resty_sha256 = require('resty.sha256') local str = require('resty.string') -local _M = { _VERSION = '0.1.2' } +local _M = { _VERSION = '0.1.3' } local function get_iso8601_basic(timestamp) return os.date('!%Y%m%dT%H%M%SZ', timestamp) @@ -17,15 +17,15 @@ end local function get_derived_signing_key(keys, timestamp, region, service) local h_date = resty_hmac:new('AWS4' .. keys['secret_key'], resty_hmac.ALGOS.SHA256) h_date:update(get_iso8601_basic_short(timestamp)) - k_date = h_date:final() + local k_date = h_date:final() local h_region = resty_hmac:new(k_date, resty_hmac.ALGOS.SHA256) h_region:update(region) - k_region = h_region:final() + local k_region = h_region:final() local h_service = resty_hmac:new(k_region, resty_hmac.ALGOS.SHA256) h_service:update(service) - k_service = h_service:final() + local k_service = h_service:final() local h = resty_hmac:new(k_service, resty_hmac.ALGOS.SHA256) h:update('aws4_request') From 13ec95515578534a66feb91017a2941580afeef1 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 13:41:05 +0100 Subject: [PATCH 03/11] Support unsigned payloads --- lib/resty/aws.lua | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index b9ba6d9..f48fd82 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -49,8 +49,13 @@ local function get_sha256_digest(s) return str.to_hex(h:final()) end -local function get_hashed_canonical_request(timestamp, host, uri) - local digest = get_sha256_digest(ngx.var.request_body) +local function get_hashed_canonical_request(timestamp, host, uri, unsigned) + local digest + if unsigned ~= nil then + digest = 'UNSIGNED-PAYLOAD' + else + digest = get_sha256_digest(ngx.var.request_body) + end local canonical_request = ngx.var.request_method .. '\n' .. uri .. '\n' .. '\n' @@ -63,11 +68,11 @@ local function get_hashed_canonical_request(timestamp, host, uri) return get_sha256_digest(canonical_request) end -local function get_string_to_sign(timestamp, region, service, host, uri) +local function get_string_to_sign(timestamp, region, service, host, uri, unsigned) return 'AWS4-HMAC-SHA256\n' .. get_iso8601_basic(timestamp) .. '\n' .. get_cred_scope(timestamp, region, service) .. '\n' - .. get_hashed_canonical_request(timestamp, host, uri) + .. get_hashed_canonical_request(timestamp, host, uri, unsigned) end local function get_signature(derived_signing_key, string_to_sign) @@ -76,9 +81,9 @@ local function get_signature(derived_signing_key, string_to_sign) return h:final(nil, true) end -local function get_authorization(keys, timestamp, region, service, host, uri) +local function get_authorization(keys, timestamp, region, service, host, uri, unsigned) local derived_signing_key = get_derived_signing_key(keys, timestamp, region, service) - local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri) + local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri, unsigned) local auth = 'AWS4-HMAC-SHA256 ' .. 'Credential=' .. keys['access_key'] .. '/' .. get_cred_scope(timestamp, region, service) .. ', SignedHeaders=' .. get_signed_headers() @@ -106,7 +111,7 @@ local function get_service_and_region(host) return nil, nil end -function _M.aws_set_headers(access_key, secret_key, host, uri, region, service) +function _M.aws_set_headers(access_key, secret_key, host, uri, region, service, unsigned) local creds = { access_key = access_key, secret_key = secret_key @@ -115,16 +120,20 @@ function _M.aws_set_headers(access_key, secret_key, host, uri, region, service) if region == nil or service == nil then service, region = get_service_and_region(host) end - local auth = get_authorization(creds, timestamp, region, service, host, uri) + local auth = get_authorization(creds, timestamp, region, service, host, uri, unsigned) ngx.req.set_header('Authorization', auth) ngx.req.set_header('Host', host) ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp)) end -function _M.s3_set_headers(access_key, secret_key, host, uri, region, service) - _M.aws_set_headers(access_key, secret_key, host, uri, region service) - ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) +function _M.s3_set_headers(access_key, secret_key, host, uri, region, service, unsigned) + _M.aws_set_headers(access_key, secret_key, host, uri, region service, unsigned) + if (unsigned ~= nil) then + ngx.req.set_header('x-amz-content-sha256', 'UNSIGNED-PAYLOAD') + else + ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) + end end -return _M +return _M \ No newline at end of file From ca3f9f0eeceace61aa20f6c45ff86d1a490868df Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 13:41:21 +0100 Subject: [PATCH 04/11] Fix typo --- lib/resty/aws.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index f48fd82..3afac30 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -128,7 +128,7 @@ function _M.aws_set_headers(access_key, secret_key, host, uri, region, service, end function _M.s3_set_headers(access_key, secret_key, host, uri, region, service, unsigned) - _M.aws_set_headers(access_key, secret_key, host, uri, region service, unsigned) + _M.aws_set_headers(access_key, secret_key, host, uri, region, service, unsigned) if (unsigned ~= nil) then ngx.req.set_header('x-amz-content-sha256', 'UNSIGNED-PAYLOAD') else From dd82dc6ced2ceb422c3c8053f59ef901793a52c8 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 14:10:33 +0100 Subject: [PATCH 05/11] Support query_strings --- lib/resty/aws.lua | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 3afac30..f64fb87 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -49,6 +49,24 @@ local function get_sha256_digest(s) return str.to_hex(h:final()) end +local function get_canonical_query_string() + local args = ngx.req.get_uri_args() + local query_string = '' + for key, val in pairs(args) do + if query_string ~= '' then + query_string = query_string .. '&' + end + + if type(val) == "table" then + query_string = query_string .. key .. '=' .. val[0] --Get the first instance of said argument, ignore the others. (Note: Maybe we should just include all instances of said parameter?) + else + query_string = query_string .. key .. '=' .. val + end + end + + return query_string +end + local function get_hashed_canonical_request(timestamp, host, uri, unsigned) local digest if unsigned ~= nil then @@ -58,7 +76,7 @@ local function get_hashed_canonical_request(timestamp, host, uri, unsigned) end local canonical_request = ngx.var.request_method .. '\n' .. uri .. '\n' - .. '\n' + .. get_canonical_query_string() .. '\n' .. 'host:' .. host .. '\n' .. 'x-amz-content-sha256:' .. digest .. '\n' .. 'x-amz-date:' .. get_iso8601_basic(timestamp) .. '\n' From 8997189ff8ea347a3722734c3496beac133a0817 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 15:13:42 +0100 Subject: [PATCH 06/11] Update README --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 148ca35..4dba9af 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,14 @@ nearly all AWS services. ## Example ```nginx + +map $request_uri $request_uri_no_parameters { + "~^(?P.*?)(\?.*)*$" $path; +} + location / { set $s3_host s3.amazonaws.com; - set $s3_uri $request_uri; + set $s3_uri $request_uri_no_parameters; access_by_lua "local aws = require 'resty.aws'; aws.s3_set_headers(ngx.var.access_key, ngx.var.secret_key, ngx.var.s3_host, ngx.var.s3_uri)"; proxy_pass https://$s3_host$s3_uri; } From 628e26d9e8632a28b8310932bab4a0941fcc0db6 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 15:19:31 +0100 Subject: [PATCH 07/11] Make unsigned payload depend on if request_body is present --- lib/resty/aws.lua | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index f64fb87..7e6110e 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -67,17 +67,17 @@ local function get_canonical_query_string() return query_string end -local function get_hashed_canonical_request(timestamp, host, uri, unsigned) +local function get_hashed_canonical_request(timestamp, host, uri) local digest - if unsigned ~= nil then + if ngx.var.request_body == nil then digest = 'UNSIGNED-PAYLOAD' else digest = get_sha256_digest(ngx.var.request_body) end - local canonical_request = ngx.var.request_method .. '\n' - .. uri .. '\n' - .. get_canonical_query_string() .. '\n' - .. 'host:' .. host .. '\n' + local canonical_request = ngx.var.request_method .. '\n' + .. uri .. '\n' + .. get_canonical_query_string() .. '\n' + .. 'host:' .. host .. '\n' .. 'x-amz-content-sha256:' .. digest .. '\n' .. 'x-amz-date:' .. get_iso8601_basic(timestamp) .. '\n' .. '\n' @@ -86,11 +86,11 @@ local function get_hashed_canonical_request(timestamp, host, uri, unsigned) return get_sha256_digest(canonical_request) end -local function get_string_to_sign(timestamp, region, service, host, uri, unsigned) +local function get_string_to_sign(timestamp, region, service, host, uri) return 'AWS4-HMAC-SHA256\n' .. get_iso8601_basic(timestamp) .. '\n' .. get_cred_scope(timestamp, region, service) .. '\n' - .. get_hashed_canonical_request(timestamp, host, uri, unsigned) + .. get_hashed_canonical_request(timestamp, host, uri) end local function get_signature(derived_signing_key, string_to_sign) @@ -99,9 +99,9 @@ local function get_signature(derived_signing_key, string_to_sign) return h:final(nil, true) end -local function get_authorization(keys, timestamp, region, service, host, uri, unsigned) +local function get_authorization(keys, timestamp, region, service, host, uri) local derived_signing_key = get_derived_signing_key(keys, timestamp, region, service) - local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri, unsigned) + local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri) local auth = 'AWS4-HMAC-SHA256 ' .. 'Credential=' .. keys['access_key'] .. '/' .. get_cred_scope(timestamp, region, service) .. ', SignedHeaders=' .. get_signed_headers() @@ -129,7 +129,7 @@ local function get_service_and_region(host) return nil, nil end -function _M.aws_set_headers(access_key, secret_key, host, uri, region, service, unsigned) +function _M.aws_set_headers(access_key, secret_key, host, uri, region, service) local creds = { access_key = access_key, secret_key = secret_key @@ -138,16 +138,16 @@ function _M.aws_set_headers(access_key, secret_key, host, uri, region, service, if region == nil or service == nil then service, region = get_service_and_region(host) end - local auth = get_authorization(creds, timestamp, region, service, host, uri, unsigned) + local auth = get_authorization(creds, timestamp, region, service, host, uri) ngx.req.set_header('Authorization', auth) ngx.req.set_header('Host', host) ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp)) end -function _M.s3_set_headers(access_key, secret_key, host, uri, region, service, unsigned) - _M.aws_set_headers(access_key, secret_key, host, uri, region, service, unsigned) - if (unsigned ~= nil) then +function _M.s3_set_headers(access_key, secret_key, host, uri, region, service) + _M.aws_set_headers(access_key, secret_key, host, uri, region, service) + if ngx.var.request_body == nil then ngx.req.set_header('x-amz-content-sha256', 'UNSIGNED-PAYLOAD') else ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) From 680e5b48e4566416a94635cc42980a368c371d79 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 15:35:18 +0100 Subject: [PATCH 08/11] Oppertunistic signing --- lib/resty/aws.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 7e6110e..9c9d0cd 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -69,12 +69,13 @@ end local function get_hashed_canonical_request(timestamp, host, uri) local digest - if ngx.var.request_body == nil then + if ngx.var.request_body == nil and ngx.var.request_method == 'PUT' then digest = 'UNSIGNED-PAYLOAD' else digest = get_sha256_digest(ngx.var.request_body) end - local canonical_request = ngx.var.request_method .. '\n' + local canonical_request = + ngx.var.request_method .. '\n' .. uri .. '\n' .. get_canonical_query_string() .. '\n' .. 'host:' .. host .. '\n' @@ -147,7 +148,7 @@ end function _M.s3_set_headers(access_key, secret_key, host, uri, region, service) _M.aws_set_headers(access_key, secret_key, host, uri, region, service) - if ngx.var.request_body == nil then + if ngx.var.request_body == nil and ngx.var.request_method == 'PUT' then ngx.req.set_header('x-amz-content-sha256', 'UNSIGNED-PAYLOAD') else ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) From c3727023112d9814198c5f762543cc48bec145db Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 16:31:44 +0100 Subject: [PATCH 09/11] Update README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4dba9af..37c6eff 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ nearly all AWS services. ```nginx +#You're limited by your max_body_size (and memory it may take up) +client_max_body_size 1G; +client_body_buffer_size 1024M; +client_body_in_single_buffer on; + map $request_uri $request_uri_no_parameters { "~^(?P.*?)(\?.*)*$" $path; } From 063757b5d6e5b107f6e216a3b3d20e095aa217b6 Mon Sep 17 00:00:00 2001 From: Isa S Date: Sun, 11 Feb 2024 17:00:35 +0100 Subject: [PATCH 10/11] Fix canonicalQueryString on multiple arguments + URI encode --- lib/resty/aws.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 9c9d0cd..9340341 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -54,13 +54,13 @@ local function get_canonical_query_string() local query_string = '' for key, val in pairs(args) do if query_string ~= '' then - query_string = query_string .. '&' + query_string = '&' .. query_string end if type(val) == "table" then - query_string = query_string .. key .. '=' .. val[0] --Get the first instance of said argument, ignore the others. (Note: Maybe we should just include all instances of said parameter?) + query_string = ngx.escape_uri(key) .. '=' .. ngx.escape_uri(val[0]) .. query_string --Get the first instance of said argument, ignore the others. (Note: Maybe we should just include all instances of said parameter?) else - query_string = query_string .. key .. '=' .. val + query_string = ngx.escape_uri(key) .. '=' .. ngx.escape_uri(val) .. query_string end end From 0b08243788ff0d847f77d0838c3338e4266544af Mon Sep 17 00:00:00 2001 From: Isa S Date: Wed, 21 Feb 2024 11:13:21 +0100 Subject: [PATCH 11/11] Handle keys with empty value (Like ?acl) --- lib/resty/aws.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/resty/aws.lua b/lib/resty/aws.lua index 9340341..610c137 100644 --- a/lib/resty/aws.lua +++ b/lib/resty/aws.lua @@ -60,7 +60,11 @@ local function get_canonical_query_string() if type(val) == "table" then query_string = ngx.escape_uri(key) .. '=' .. ngx.escape_uri(val[0]) .. query_string --Get the first instance of said argument, ignore the others. (Note: Maybe we should just include all instances of said parameter?) else - query_string = ngx.escape_uri(key) .. '=' .. ngx.escape_uri(val) .. query_string + if val == true then + query_string = ngx.escape_uri(key) .. '=' .. query_string + else + query_string = ngx.escape_uri(key) .. '=' .. ngx.escape_uri(val) .. query_string + end end end