From eb23a2dbbf8d958a7e007b68b6a11a55a4aa568b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Sun, 17 Dec 2023 16:00:17 +0100 Subject: [PATCH 1/7] Add tests for retry logic based on response status code --- Project.toml | 6 +- test/azure_blobs_exception_tests.jl | 140 +++++++++++++++++++++++++++- test/common_testsetup.jl | 4 +- test/runtests.jl | 4 +- 4 files changed, 148 insertions(+), 6 deletions(-) diff --git a/Project.toml b/Project.toml index a9fa39c..176bebb 100644 --- a/Project.toml +++ b/Project.toml @@ -4,14 +4,18 @@ version = "0.1.0" [compat] CloudBase = "1" +HTTP = "1" ReTestItems = "1" +Sockets = "1" Test = "1" julia = "1.8" [extras] CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["CloudBase", "ReTestItems", "Test"] +test = ["CloudBase", "HTTP", "ReTestItems", "Sockets", "Test"] diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index 7752fd4..dde1232 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -41,7 +41,7 @@ @test false # Should have thrown an error catch e @test e isa ErrorException - @test occursin("400 Bad Request", e.msg) + @test occursin("400 Bad Request", e.msg) # Should this be 403 Forbidden? We've seen that with invalid SAS tokens @test occursin("Authentication information is not given in the correct format", e.msg) end @@ -151,3 +151,141 @@ @test res == 1 # Rust CResult::Error end end # @testitem + +@testitem "BlobStorage retries" setup=[InitializeRustStore] begin + using CloudBase.CloudTest: Azurite + import CloudBase + using ObjectStore: blob_get!, blob_put, AzureCredentials + import HTTP + import Sockets + + max_retries = InitializeRustStore.max_retries + + function test_status(method, response_status, headers=nothing) + @assert method === :GET || method === :PUT + nretries = Ref(0) + response_body = "response body from the dummy server" + account = "myaccount" + container = "mycontainer" + shared_key_from_azurite = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + (port, tcp_server) = Sockets.listenany(8081) + http_server = HTTP.serve!(tcp_server) do request::HTTP.Request + if request.method == "GET" && request.target == "/$account/$container/_this_file_does_not_exist" + # This is the exploratory ping from connect_and_test in lib.rs + return HTTP.Response(404, "Yup, still doesn't exist") + end + nretries[] += 1 + response = isnothing(headers) ? HTTP.Response(response_status, response_body) : HTTP.Response(response_status, headers, response_body) + return response + end + + baseurl = "http://127.0.0.1:$port/$account/$container/" + creds = AzureCredentials(account, container, shared_key_from_azurite, baseurl) + + try + method === :GET && blob_get!(joinpath(baseurl, "blob"), zeros(UInt8, 5), creds) + method === :PUT && blob_put(joinpath(baseurl, "blob"), codeunits("a,b,c"), creds) + @test false # Should have thrown an error + catch e + @test e isa ErrorException + @test occursin(string(response_status), e.msg) + response_status < 500 && (@test occursin("response body from the dummy server", e.msg)) + finally + close(http_server) + end + wait(http_server) + return nretries[] + end + + # See https://learn.microsoft.com/en-us/rest/api/searchservice/http-status-codes + + @testset "400: Bad Request" begin + # Returned when there's an error in the request URI, headers, or body. The response body + # contains an error message explaining what the specific problem is. + nretries = test_status(:GET, 400) + @test nretries == 1 broken=true + nretries = test_status(:PUT, 400) + @test nretries == 1 broken=true + end + + @testset "403: Forbidden" begin + # Returned when you pass an invalid api-key. + nretries = test_status(:GET, 403) + @test nretries == 1 broken=true + nretries = test_status(:PUT, 403) + @test nretries == 1 broken=true + end + + @testset "404: Not Found" begin + nretries = test_status(:GET, 404) + @test nretries == 1 + end + + @testset "405: Method Not Supported" begin + nretries = test_status(:GET, 405, ["Allow" => "PUT"]) + @test nretries == 1 broken=true + nretries = test_status(:PUT, 405, ["Allow" => "GET"]) + @test nretries == 1 broken=true + end + + @testset "409: Conflict" begin + # Returned when write operations conflict. + # NOTE: We currently don't retry but maybe we should? This is probably a case where the + # retry logic should add more noise to the backoff so that multiple writers don't collide on retry. + nretries = test_status(:GET, 409) + @test nretries == max_retries broken=true + nretries = test_status(:PUT, 409) + @test nretries == max_retries broken=true + end + + @testset "412: Precondition Failed" begin + # Returned when an If-Match or If-None-Match header's condition evaluates to false + nretries = test_status(:GET, 412) + @test nretries == 1 + nretries = test_status(:PUT, 412) + @test nretries == 1 + end + + @testset "413: Content Too Large" begin + # https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob?tabs=shared-access-signatures#remarks + nretries = test_status(:PUT, 413) + @test nretries == 1 broken=true + end + + @testset "429: Too Many Requests" begin + # TODO: We probably should respect the Retry-After header, but we currently don't + # (and we don't know if Azure actually sets it) + # NOTE: This can happen when Azure is throttling us, so it might be a good idea to retry with some + # larger initial backoff (very eager retries probably only make the situation worse). + nretries = test_status(:GET, 429, ["Retry-After" => 10]) + @test nretries == max_retries broken=true + nretries = test_status(:PUT, 429, ["Retry-After" => 10]) + @test nretries == max_retries broken=true + end + + @testset "502: Bad Gateway" begin + # This error occurs when you enter HTTP instead of HTTPS in the connection. + nretries = test_status(:GET, 502) + @test nretries == 1 broken=true + nretries = test_status(:PUT, 502) + @test nretries == 1 broken=true + end + + @testset "503: Service Unavailable" begin + # NOTE: This seems similar to 429 and the Azure docs specifically say: + # Important: In this case, we highly recommend that your client code back off and wait before retrying + nretries = test_status(:GET, 503) + @test nretries == max_retries broken=true + nretries = test_status(:PUT, 503) + @test nretries == max_retries broken=true + end + + @testset "504: Gateway Timeout" begin + # Azure AI Search listens on HTTPS port 443. If your search service URL contains HTTP instead of HTTPS, a 504 status code is returned. + nretries = test_status(:GET, 504) + @test nretries == 1 broken=true + nretries = test_status(:PUT, 504) + @test nretries == 1 broken=true + end +end diff --git a/test/common_testsetup.jl b/test/common_testsetup.jl index d55afa2..650fdec 100644 --- a/test/common_testsetup.jl +++ b/test/common_testsetup.jl @@ -2,7 +2,7 @@ using ObjectStore # Since we currently only support centralized configs, we need to have one that is compatible # with all the tests (some of the tests would take too long if we use default values). - max_retries = 5 - retry_timeout_sec = 5 + max_retries = 2 + retry_timeout_sec = 2 init_object_store(ObjectStoreConfig(max_retries, retry_timeout_sec)) end diff --git a/test/runtests.jl b/test/runtests.jl index bb66c28..9cd700e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,5 +2,5 @@ using ReTestItems using ObjectStore withenv("RUST_BACKTRACE"=>1) do - runtests(ObjectStore, testitem_timeout=120, nworkers=1) -end \ No newline at end of file + runtests(ObjectStore, testitem_timeout=180, nworkers=1) +end From 80a2227801ee563302c7515d3f64b46216da49b1 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 19 Dec 2023 17:38:17 +0000 Subject: [PATCH 2/7] Mark some tests fixed --- test/azure_blobs_exception_tests.jl | 86 ++++++++++++++--------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index dde1232..a6328a6 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -163,7 +163,7 @@ end # @testitem function test_status(method, response_status, headers=nothing) @assert method === :GET || method === :PUT - nretries = Ref(0) + nrequests = Ref(0) response_body = "response body from the dummy server" account = "myaccount" container = "mycontainer" @@ -175,7 +175,7 @@ end # @testitem # This is the exploratory ping from connect_and_test in lib.rs return HTTP.Response(404, "Yup, still doesn't exist") end - nretries[] += 1 + nrequests[] += 1 response = isnothing(headers) ? HTTP.Response(response_status, response_body) : HTTP.Response(response_status, headers, response_body) return response end @@ -195,7 +195,7 @@ end # @testitem close(http_server) end wait(http_server) - return nretries[] + return nrequests[] end # See https://learn.microsoft.com/en-us/rest/api/searchservice/http-status-codes @@ -203,54 +203,54 @@ end # @testitem @testset "400: Bad Request" begin # Returned when there's an error in the request URI, headers, or body. The response body # contains an error message explaining what the specific problem is. - nretries = test_status(:GET, 400) - @test nretries == 1 broken=true - nretries = test_status(:PUT, 400) - @test nretries == 1 broken=true + nrequests = test_status(:GET, 400) + @test nrequests == 1 + nrequests = test_status(:PUT, 400) + @test nrequests == 1 end @testset "403: Forbidden" begin # Returned when you pass an invalid api-key. - nretries = test_status(:GET, 403) - @test nretries == 1 broken=true - nretries = test_status(:PUT, 403) - @test nretries == 1 broken=true + nrequests = test_status(:GET, 403) + @test nrequests == 1 + nrequests = test_status(:PUT, 403) + @test nrequests == 1 end @testset "404: Not Found" begin - nretries = test_status(:GET, 404) - @test nretries == 1 + nrequests = test_status(:GET, 404) + @test nrequests == 1 end @testset "405: Method Not Supported" begin - nretries = test_status(:GET, 405, ["Allow" => "PUT"]) - @test nretries == 1 broken=true - nretries = test_status(:PUT, 405, ["Allow" => "GET"]) - @test nretries == 1 broken=true + nrequests = test_status(:GET, 405, ["Allow" => "PUT"]) + @test nrequests == 1 + nrequests = test_status(:PUT, 405, ["Allow" => "GET"]) + @test nrequests == 1 end @testset "409: Conflict" begin # Returned when write operations conflict. # NOTE: We currently don't retry but maybe we should? This is probably a case where the # retry logic should add more noise to the backoff so that multiple writers don't collide on retry. - nretries = test_status(:GET, 409) - @test nretries == max_retries broken=true - nretries = test_status(:PUT, 409) - @test nretries == max_retries broken=true + nrequests = test_status(:GET, 409) + @test nrequests == 1 + max_retries broken=true + nrequests = test_status(:PUT, 409) + @test nrequests == 1 + max_retries broken=true end @testset "412: Precondition Failed" begin # Returned when an If-Match or If-None-Match header's condition evaluates to false - nretries = test_status(:GET, 412) - @test nretries == 1 - nretries = test_status(:PUT, 412) - @test nretries == 1 + nrequests = test_status(:GET, 412) + @test nrequests == 1 + nrequests = test_status(:PUT, 412) + @test nrequests == 1 end @testset "413: Content Too Large" begin # https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob?tabs=shared-access-signatures#remarks - nretries = test_status(:PUT, 413) - @test nretries == 1 broken=true + nrequests = test_status(:PUT, 413) + @test nrequests == 1 end @testset "429: Too Many Requests" begin @@ -258,34 +258,34 @@ end # @testitem # (and we don't know if Azure actually sets it) # NOTE: This can happen when Azure is throttling us, so it might be a good idea to retry with some # larger initial backoff (very eager retries probably only make the situation worse). - nretries = test_status(:GET, 429, ["Retry-After" => 10]) - @test nretries == max_retries broken=true - nretries = test_status(:PUT, 429, ["Retry-After" => 10]) - @test nretries == max_retries broken=true + nrequests = test_status(:GET, 429, ["Retry-After" => 10]) + @test nrequests == 1 + max_retries broken=true + nrequests = test_status(:PUT, 429, ["Retry-After" => 10]) + @test nrequests == 1 + max_retries broken=true end @testset "502: Bad Gateway" begin # This error occurs when you enter HTTP instead of HTTPS in the connection. - nretries = test_status(:GET, 502) - @test nretries == 1 broken=true - nretries = test_status(:PUT, 502) - @test nretries == 1 broken=true + nrequests = test_status(:GET, 502) + @test nrequests == 1 broken=true + nrequests = test_status(:PUT, 502) + @test nrequests == 1 broken=true end @testset "503: Service Unavailable" begin # NOTE: This seems similar to 429 and the Azure docs specifically say: # Important: In this case, we highly recommend that your client code back off and wait before retrying - nretries = test_status(:GET, 503) - @test nretries == max_retries broken=true - nretries = test_status(:PUT, 503) - @test nretries == max_retries broken=true + nrequests = test_status(:GET, 503) + @test nrequests == 1 + max_retries + nrequests = test_status(:PUT, 503) + @test nrequests == 1 + max_retries end @testset "504: Gateway Timeout" begin # Azure AI Search listens on HTTPS port 443. If your search service URL contains HTTP instead of HTTPS, a 504 status code is returned. - nretries = test_status(:GET, 504) - @test nretries == 1 broken=true - nretries = test_status(:PUT, 504) - @test nretries == 1 broken=true + nrequests = test_status(:GET, 504) + @test nrequests == 1 broken=true + nrequests = test_status(:PUT, 504) + @test nrequests == 1 broken=true end end From 8a3be8cf392ca1b5daa460d785f64f87d0990287 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 19 Dec 2023 19:05:18 +0000 Subject: [PATCH 3/7] Add comments linking to relevent docs --- test/azure_blobs_exception_tests.jl | 62 ++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index a6328a6..91e9b73 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -152,6 +152,15 @@ end end # @testitem +### See Azure Blob Storage docs: https://learn.microsoft.com/en-us/rest/api/storageservices +### - "Common REST API error codes": +### https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes +### - "Azure Blob Storage error codes": +### https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes +### - "Get Blob" +### https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob +### - "Put Blob" +### https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob @testitem "BlobStorage retries" setup=[InitializeRustStore] begin using CloudBase.CloudTest: Azurite import CloudBase @@ -198,11 +207,11 @@ end # @testitem return nrequests[] end - # See https://learn.microsoft.com/en-us/rest/api/searchservice/http-status-codes - @testset "400: Bad Request" begin # Returned when there's an error in the request URI, headers, or body. The response body # contains an error message explaining what the specific problem is. + # See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes + # See https://www.rfc-editor.org/rfc/rfc9110#status.400 nrequests = test_status(:GET, 400) @test nrequests == 1 nrequests = test_status(:PUT, 400) @@ -211,6 +220,7 @@ end # @testitem @testset "403: Forbidden" begin # Returned when you pass an invalid api-key. + # See https://www.rfc-editor.org/rfc/rfc9110#status.403 nrequests = test_status(:GET, 403) @test nrequests == 1 nrequests = test_status(:PUT, 403) @@ -218,11 +228,15 @@ end # @testitem end @testset "404: Not Found" begin + # Returned when container not found or blob not found + # See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes + # See https://www.rfc-editor.org/rfc/rfc9110#status.404 nrequests = test_status(:GET, 404) @test nrequests == 1 end @testset "405: Method Not Supported" begin + # See https://www.rfc-editor.org/rfc/rfc9110#status.405 nrequests = test_status(:GET, 405, ["Allow" => "PUT"]) @test nrequests == 1 nrequests = test_status(:PUT, 405, ["Allow" => "GET"]) @@ -231,7 +245,9 @@ end # @testitem @testset "409: Conflict" begin # Returned when write operations conflict. - # NOTE: We currently don't retry but maybe we should? This is probably a case where the + # See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes + # See https://www.rfc-editor.org/rfc/rfc9110#status.409 + # TODO: We currently don't retry but maybe we should? This is probably a case where the # retry logic should add more noise to the backoff so that multiple writers don't collide on retry. nrequests = test_status(:GET, 409) @test nrequests == 1 + max_retries broken=true @@ -241,6 +257,8 @@ end # @testitem @testset "412: Precondition Failed" begin # Returned when an If-Match or If-None-Match header's condition evaluates to false + # See https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob#blob-custom-properties + # See https://www.rfc-editor.org/rfc/rfc9110#status.412 nrequests = test_status(:GET, 412) @test nrequests == 1 nrequests = test_status(:PUT, 412) @@ -248,15 +266,23 @@ end # @testitem end @testset "413: Content Too Large" begin - # https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob?tabs=shared-access-signatures#remarks + # See https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob#remarks + # If you attempt to upload either a block blob that's larger than the maximum + # permitted size for that service version or a page blob that's larger than 8 TiB, + # the service returns status code 413 (Request Entity Too Large). Blob Storage also + # returns additional information about the error in the response, including the + # maximum permitted blob size, in bytes. + # See https://www.rfc-editor.org/rfc/rfc9110#status.413 nrequests = test_status(:PUT, 413) @test nrequests == 1 end @testset "429: Too Many Requests" begin + # See https://www.rfc-editor.org/rfc/rfc6585#section-4 + # See https://www.rfc-editor.org/rfc/rfc9110#field.retry-after # TODO: We probably should respect the Retry-After header, but we currently don't # (and we don't know if Azure actually sets it) - # NOTE: This can happen when Azure is throttling us, so it might be a good idea to retry with some + # This can happen when Azure is throttling us, so it might be a good idea to retry with some # larger initial backoff (very eager retries probably only make the situation worse). nrequests = test_status(:GET, 429, ["Retry-After" => 10]) @test nrequests == 1 + max_retries broken=true @@ -265,7 +291,11 @@ end # @testitem end @testset "502: Bad Gateway" begin - # This error occurs when you enter HTTP instead of HTTPS in the connection. + # https://www.rfc-editor.org/rfc/rfc9110#status.502 + # The 502 (Bad Gateway) status code indicates that the server, while acting as a + # gateway or proxy, received an invalid response from an inbound server it accessed + # while attempting to fulfill the request. + # This error can occur when you enter HTTP instead of HTTPS in the connection. nrequests = test_status(:GET, 502) @test nrequests == 1 broken=true nrequests = test_status(:PUT, 502) @@ -273,8 +303,19 @@ end # @testitem end @testset "503: Service Unavailable" begin - # NOTE: This seems similar to 429 and the Azure docs specifically say: - # Important: In this case, we highly recommend that your client code back off and wait before retrying + # See https://www.rfc-editor.org/rfc/rfc9110#status.503 + # The 503 (Service Unavailable) status code indicates that the server is currently + # unable to handle the request due to a temporary overload or scheduled maintenance, + # which will likely be alleviated after some delay. The server MAY send a Retry-After + # header field (Section 10.2.3) to suggest an appropriate amount of time for the + # client to wait before retrying the request. + # See https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes + # An operation on any of the Azure Storage services can return the following error codes: + # Error code HTTP status code User message + # ServerBusy Service Unavailable (503) The server is currently unable to receive requests. Please retry your request. + # ServerBusy Service Unavailable (503) Ingress is over the account limit. + # ServerBusy Service Unavailable (503) Egress is over the account limit. + # ServerBusy Service Unavailable (503) Operations per second is over the account limit. nrequests = test_status(:GET, 503) @test nrequests == 1 + max_retries nrequests = test_status(:PUT, 503) @@ -282,7 +323,10 @@ end # @testitem end @testset "504: Gateway Timeout" begin - # Azure AI Search listens on HTTPS port 443. If your search service URL contains HTTP instead of HTTPS, a 504 status code is returned. + # See https://www.rfc-editor.org/rfc/rfc9110#status.504 + # The 504 (Gateway Timeout) status code indicates that the server, while acting as + # a gateway or proxy, did not receive a timely response from an upstream server it + # needed to access in order to complete the request nrequests = test_status(:GET, 504) @test nrequests == 1 broken=true nrequests = test_status(:PUT, 504) From c1084984844da3a3fbde7fe68285f4275dfebed9 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 19 Dec 2023 19:06:27 +0000 Subject: [PATCH 4/7] Retry 503 and 504 since they can be transient errors --- test/azure_blobs_exception_tests.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index 91e9b73..1b03baa 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -297,9 +297,9 @@ end # @testitem # while attempting to fulfill the request. # This error can occur when you enter HTTP instead of HTTPS in the connection. nrequests = test_status(:GET, 502) - @test nrequests == 1 broken=true + @test nrequests == 1 + max_retries nrequests = test_status(:PUT, 502) - @test nrequests == 1 broken=true + @test nrequests == 1 + max_retries end @testset "503: Service Unavailable" begin @@ -328,8 +328,8 @@ end # @testitem # a gateway or proxy, did not receive a timely response from an upstream server it # needed to access in order to complete the request nrequests = test_status(:GET, 504) - @test nrequests == 1 broken=true + @test nrequests == 1 + max_retries nrequests = test_status(:PUT, 504) - @test nrequests == 1 broken=true + @test nrequests == 1 + max_retries end end From b5a8b0215240dfe08783f5ac96a67060bf7a4775 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 19 Dec 2023 19:09:01 +0000 Subject: [PATCH 5/7] Don't retry 409 None of the listed reasons seem like they could be transient https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes --- test/azure_blobs_exception_tests.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index 1b03baa..5bf79a3 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -247,12 +247,10 @@ end # @testitem # Returned when write operations conflict. # See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes # See https://www.rfc-editor.org/rfc/rfc9110#status.409 - # TODO: We currently don't retry but maybe we should? This is probably a case where the - # retry logic should add more noise to the backoff so that multiple writers don't collide on retry. nrequests = test_status(:GET, 409) - @test nrequests == 1 + max_retries broken=true + @test nrequests == 1 nrequests = test_status(:PUT, 409) - @test nrequests == 1 + max_retries broken=true + @test nrequests == 1 end @testset "412: Precondition Failed" begin From ed43a99de8ebdf8f77728fc7b8dfeafd5faa29c5 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 20 Dec 2023 16:03:15 +0000 Subject: [PATCH 6/7] Test current 429 behaviour --- test/azure_blobs_exception_tests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index 5bf79a3..c00f66e 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -277,6 +277,10 @@ end # @testitem @testset "429: Too Many Requests" begin # See https://www.rfc-editor.org/rfc/rfc6585#section-4 + nrequests = test_status(:GET, 429) + @test nrequests == 1 + nrequests = test_status(:PUT, 429) + @test nrequests == 1 # See https://www.rfc-editor.org/rfc/rfc9110#field.retry-after # TODO: We probably should respect the Retry-After header, but we currently don't # (and we don't know if Azure actually sets it) From 53379b7eea97ee64f6e2e477d5fef043e33ca62e Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 20 Dec 2023 16:13:38 +0000 Subject: [PATCH 7/7] Fixup test setup name --- test/azure_blobs_exception_tests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/azure_blobs_exception_tests.jl b/test/azure_blobs_exception_tests.jl index c00f66e..5681b8e 100644 --- a/test/azure_blobs_exception_tests.jl +++ b/test/azure_blobs_exception_tests.jl @@ -161,14 +161,14 @@ end # @testitem ### https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob ### - "Put Blob" ### https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob -@testitem "BlobStorage retries" setup=[InitializeRustStore] begin +@testitem "BlobStorage retries" setup=[InitializeObjectStore] begin using CloudBase.CloudTest: Azurite import CloudBase using ObjectStore: blob_get!, blob_put, AzureCredentials import HTTP import Sockets - max_retries = InitializeRustStore.max_retries + max_retries = InitializeObjectStore.max_retries function test_status(method, response_status, headers=nothing) @assert method === :GET || method === :PUT