[CI] GoogleCloudStorageThirdPartyTests testCreateSnapshot failing

[tlrx]

Failed twice in last 24 hours:
https://gradle-enterprise.elastic.co/s/yt5q2acttt5ci
https://gradle-enterprise.elastic.co/s/2jkfs57fcye66

Build scan:
https://gradle-enterprise.elastic.co/s/2jkfs57fcye66/tests/:plugins:repository-gcs:gcsThirdPartyTest/org.elasticsearch.repositories.gcs.GoogleCloudStorageThirdPartyTests/testCreateSnapshot

Reproduction line:
./gradlew ':plugins:repository-gcs:gcsThirdPartyTest' --tests "org.elasticsearch.repositories.gcs.GoogleCloudStorageThirdPartyTests.testCreateSnapshot" -Dtests.seed=21DCBF3B6D9F3C04 -Dtests.locale=ar-TN -Dtests.timezone=Asia/Bahrain -Druntime.java=8

Applicable branches:
7.x

Reproduces locally?:
No

Failure history:
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.repositories.gcs.GoogleCloudStorageThirdPartyTests&tests.test=testCreateSnapshot

Failure excerpt:

org.elasticsearch.snapshots.SnapshotException: [test-repo:test-snap-1625044852548/AsPCok_YSNqBXJ2ZhI807g] failed to get snapshot infotest-snap-1625044852548/AsPCok_YSNqBXJ2ZhI807g

  at __randomizedtesting.SeedInfo.seed([21DCBF3B6D9F3C04:DFBA2A885B2CCE1D]:0)
  at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$getOneSnapshotInfo$51(BlobStoreRepository.java:1508)
  at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  at java.lang.Thread.run(Thread.java:748)

  Caused by: org.elasticsearch.gateway.CorruptStateException: checksums do not match read [3330500159] but expected [4257216641]

    at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat$DeserializeMetaBlobInputStream.verifyFooter(ChecksumBlobStoreFormat.java:226)
    at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.deserialize(ChecksumBlobStoreFormat.java:121)
    at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.read(ChecksumBlobStoreFormat.java:87)
    at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$getOneSnapshotInfo$51(BlobStoreRepository.java:1504)
    at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)

[elasticmachine]

Pinging @elastic/es-distributed (Team:Distributed)

[tlrx]

Another test failed in the same build with the following exception:

» [2021-06-30T09:21:22,059][WARN ][o.e.s.SnapshotShardsService] [integTest-0] [[eqoohgirdo][2]][repository:searchable-snapshot/za_Y52XsSMW4T9YSbNdRHg] failed to snapshot shard
» com.google.cloud.storage.StorageException: Socket Closed
» at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:231) ~[?:?]
» at com.google.cloud.storage.spi.v1.HttpStorageRpc.writeWithResponse(HttpStorageRpc.java:822) ~[?:?]
» at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:69) ~[?:?]
» at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_291]
» at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) ~[?:?]
» at com.google.cloud.RetryHelper.run(RetryHelper.java:76) ~[?:?]
» at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) ~[?:?]
» at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:61) ~[?:?]
» at com.google.cloud.BaseWriteChannel.close(BaseWriteChannel.java:151) ~[?:?]
» at org.elasticsearch.repositories.gcs.SocketAccess.lambda$doPrivilegedVoidIOException$0(SocketAccess.java:43) ~[?:?]
» at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_291]
» at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedVoidIOException(SocketAccess.java:42) ~[?:?]
» at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:293) ~[?:?]
» at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:86) ~[?:?]
» at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.write(ChecksumBlobStoreFormat.java:273) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$snapshotShard$77(BlobStoreRepository.java:2786) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListenerDirectly(ListenableFuture.java:113) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:100) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:139) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:52) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:55) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.ActionListener$DelegatingActionListener.onResponse(ActionListener.java:184) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.repositories.blobstore.BlobStoreRepository.executeOneFileSnapshot(BlobStoreRepository.java:2827) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$executeOneFileSnapshot$78(BlobStoreRepository.java:2832) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
» at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
» at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
» at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
» Caused by: java.io.IOException: Socket Closed
» at java.net.AbstractPlainSocketImpl.getInputStream(AbstractPlainSocketImpl.java:547) ~[?:1.8.0_291]
» at java.net.Socket$2.run(Socket.java:931) ~[?:1.8.0_291]
» at java.net.Socket$2.run(Socket.java:929) ~[?:1.8.0_291]
» at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_291]
» at java.net.Socket.getInputStream(Socket.java:928) ~[?:1.8.0_291]
» at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:673) ~[?:?]
» at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1593) ~[?:?]
» at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1498) ~[?:?]
» at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) ~[?:1.8.0_291]
» at com.google.api.client.http.javanet.NetHttpResponse.(NetHttpResponse.java:36) ~[?:?]
» at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:149) ~[?:?]
» at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:84) ~[?:?]
» at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1012) ~[?:?]
» at com.google.cloud.storage.spi.v1.HttpStorageRpc.writeWithResponse(HttpStorageRpc.java:792) ~[?:?]
» ... 31 more

[original-brownbear]

It's always JDK8 with these failures ... trying to find out where we break the http client or server of that version now :)

[original-brownbear]

This is actually just a repeat of #53119 now that we use resumable writes more often (previously it seems like these tests wrote so few resumeable blobs that it wasn't an issue). I wonder though if maybe this time around we can hunt down the JDK8 issue. Interestingly enough it appears as though we also see some broken blobs here which is new.

[tlrx]

Another GCS test on JDK8 also failed (https://gradle-enterprise.elastic.co/s/nwuik747fkro4):

[2021-06-30T12:29:18,433][WARN ][o.e.s.SnapshotsService   ] [integTest-0] [jygctuhstn:zonuuprlxm/5it9MeKyQ6OkFrggy5Iipw] failed to finalize snapshot
»  org.elasticsearch.snapshots.SnapshotException: [jygctuhstn:zonuuprlxm/5it9MeKyQ6OkFrggy5Iipw] failed to update snapshot in repository
»  	at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$finalizeSnapshot$42(BlobStoreRepository.java:1342) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:142) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.ActionListener$Delegating.onFailure(ActionListener.java:66) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.support.GroupedActionListener.onResponse(GroupedActionListener.java:52) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:35) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_291]
»  	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_291]
»  	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_291]
»  Caused by: com.google.cloud.storage.StorageException: Socket Closed
»  	at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:231) ~[?:?]
»  	at com.google.cloud.storage.spi.v1.HttpStorageRpc.writeWithResponse(HttpStorageRpc.java:822) ~[?:?]
»  	at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:69) ~[?:?]
»  	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_291]
»  	at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) ~[?:?]
»  	at com.google.cloud.RetryHelper.run(RetryHelper.java:76) ~[?:?]
»  	at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) ~[?:?]
»  	at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:61) ~[?:?]
»  	at com.google.cloud.BaseWriteChannel.close(BaseWriteChannel.java:151) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.SocketAccess.lambda$doPrivilegedVoidIOException$0(SocketAccess.java:43) ~[?:?]
»  	at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_291]
»  	at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedVoidIOException(SocketAccess.java:42) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:293) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:86) ~[?:?]
»  	at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.write(ChecksumBlobStoreFormat.java:273) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$finalizeSnapshot$47(BlobStoreRepository.java:1447) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:34) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	... 5 more
»  Caused by: java.io.IOException: Socket Closed
»  	at java.net.AbstractPlainSocketImpl.getInputStream(AbstractPlainSocketImpl.java:547) ~[?:1.8.0_291]
»  	at java.net.Socket$2.run(Socket.java:931) ~[?:1.8.0_291]
»  	at java.net.Socket$2.run(Socket.java:929) ~[?:1.8.0_291]
»  	at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_291]
»  	at java.net.Socket.getInputStream(Socket.java:928) ~[?:1.8.0_291]
»  	at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:673) ~[?:?]
»  	at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1593) ~[?:?]
»  	at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1498) ~[?:?]
»  	at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) ~[?:1.8.0_291]
»  	at com.google.api.client.http.javanet.NetHttpResponse.<init>(NetHttpResponse.java:36) ~[?:?]
»  	at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:149) ~[?:?]
»  	at com.google.api.client.http.javanet.NetHttpRequest.execute(NetHttpRequest.java:84) ~[?:?]
»  	at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1012) ~[?:?]
»  	at com.google.cloud.storage.spi.v1.HttpStorageRpc.writeWithResponse(HttpStorageRpc.java:792) ~[?:?]
»  	at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:69) ~[?:?]
»  	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_291]
»  	at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) ~[?:?]
»  	at com.google.cloud.RetryHelper.run(RetryHelper.java:76) ~[?:?]
»  	at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) ~[?:?]
»  	at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:61) ~[?:?]
»  	at com.google.cloud.BaseWriteChannel.close(BaseWriteChannel.java:151) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.SocketAccess.lambda$doPrivilegedVoidIOException$0(SocketAccess.java:43) ~[?:?]
»  	at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_291]
»  	at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedVoidIOException(SocketAccess.java:42) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:293) ~[?:?]
»  	at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:86) ~[?:?]
»  	at org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.write(ChecksumBlobStoreFormat.java:273) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.repositories.blobstore.BlobStoreRepository.lambda$finalizeSnapshot$47(BlobStoreRepository.java:1447) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	at org.elasticsearch.action.ActionRunnable$1.doRun(ActionRunnable.java:34) ~[elasticsearch-7.14.0-SNAPSHOT.jar:7.14.0-SNAPSHOT]
»  	... 5 more