[CI] GoogleCloudStorageBlobContainerRetriesTests.testWriteLargeBlob #52430
Description
There are two similar failures for this test, but I am not 100% sure this is network related or a test failure, but opening this one for further investigation.
This one from today: https://gradle-enterprise.elastic.co/s/wpojzqhezgkhm/console-log#L4309
This one from Feb 9th: https://gradle-enterprise.elastic.co/s/tk2o44alzkal2/console-log#L4324
Both report a Socket closed exception:
2> REPRODUCE WITH: ./gradlew ':plugins:repository-gcs:test' --tests "org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests.testWriteLargeBlob" -Dtests.seed=150CA2779F460C65 -Dtests.security.manager=true -Dtests.locale=nl-NL -Dtests.timezone=Asia/Muscat -Dcompiler.java=13 |
-- | --
| 2> com.google.cloud.storage.StorageException: Socket Closed |
| at __randomizedtesting.SeedInfo.seed([150CA2779F460C65:6A599A34EC9E7FDD]:0) |
| at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:227) |
| at com.google.cloud.storage.spi.v1.HttpStorageRpc.write(HttpStorageRpc.java:762) |
| at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:60) |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) |
| at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) |
| at com.google.cloud.RetryHelper.run(RetryHelper.java:76) |
| at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) |
| at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:53) |
| at com.google.cloud.BaseWriteChannel.flush(BaseWriteChannel.java:112) |
| at com.google.cloud.BaseWriteChannel.write(BaseWriteChannel.java:139) |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) |
| at java.nio.channels.Channels.writeFully(Channels.java:101) |
| at java.nio.channels.Channels.access$000(Channels.java:61) |
| at java.nio.channels.Channels$1.write(Channels.java:174) |
| at org.elasticsearch.core.internal.io.Streams.copy(Streams.java:55) |
| at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.lambda$writeBlobResumable$6(GoogleCloudStorageBlobStore.java:246) |
| at org.elasticsearch.repositories.gcs.SocketAccess.lambda$doPrivilegedVoidIOException$0(SocketAccess.java:54) |
| at java.security.AccessController.doPrivileged(Native Method) |
| at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedVoidIOException(SocketAccess.java:53) |
| at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlobResumable(GoogleCloudStorageBlobStore.java:245) |
| at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:216) |
| at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:67) |
| at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests.testWriteLargeBlob(GoogleCloudStorageBlobContainerRetriesTests.java:413)
But today's one has something more, from the GCP side:
1> Suppressed: com.google.cloud.storage.StorageException: 410 Gone |
-- | --
| 1> at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:227) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> at com.google.cloud.storage.spi.v1.HttpStorageRpc.write(HttpStorageRpc.java:762) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> at com.google.cloud.storage.BlobWriteChannel$1.run(BlobWriteChannel.java:60) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_241] |
| 1> at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105) ~[gax-1.45.0.jar:1.45.0] |
| 1> at com.google.cloud.RetryHelper.run(RetryHelper.java:76) ~[google-cloud-core-1.77.0.jar:1.77.0] |
| 1> at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50) ~[google-cloud-core-1.77.0.jar:1.77.0] |
| 1> at com.google.cloud.storage.BlobWriteChannel.flushBuffer(BlobWriteChannel.java:53) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> at com.google.cloud.BaseWriteChannel.close(BaseWriteChannel.java:151) ~[google-cloud-core-1.77.0.jar:1.77.0] |
| 1> at java.nio.channels.Channels$1.close(Channels.java:178) ~[?:1.8.0_241] |
| 1> at org.elasticsearch.core.internal.io.IOUtils.close(IOUtils.java:104) ~[elasticsearch-core-7.7.0-SNAPSHOT.jar:7.7.0-SNAPSHOT] |
| 1> at org.elasticsearch.core.internal.io.IOUtils.close(IOUtils.java:74) ~[elasticsearch-core-7.7.0-SNAPSHOT.jar:7.7.0-SNAPSHOT] |
| 1> at org.elasticsearch.core.internal.io.Streams.copy(Streams.java:64) ~[elasticsearch-core-7.7.0-SNAPSHOT.jar:7.7.0-SNAPSHOT] |
| 1> at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.lambda$writeBlobResumable$6(GoogleCloudStorageBlobStore.java:246) ~[main/:?] |
| 1> at org.elasticsearch.repositories.gcs.SocketAccess.lambda$doPrivilegedVoidIOException$0(SocketAccess.java:54) ~[main/:?] |
| 1> at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_241] |
| 1> at org.elasticsearch.repositories.gcs.SocketAccess.doPrivilegedVoidIOException(SocketAccess.java:53) ~[main/:?] |
| 1> at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlobResumable(GoogleCloudStorageBlobStore.java:245) [main/:?] |
| 1> at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.writeBlob(GoogleCloudStorageBlobStore.java:216) [main/:?] |
| 1> at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainer.writeBlob(GoogleCloudStorageBlobContainer.java:67) [main/:?] |
| 1> at org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests.testWriteLargeBlob(GoogleCloudStorageBlobContainerRetriesTests.java:413) [test/:?] |
| 1> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] |
| 1> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?] |
| 1> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] |
| 1> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_241] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequire 1> d.java:53) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54) [lucene-test-framework-8.5.0-snapshot-d62f6307658.jar:8.5.0-snapshot-d62f6307658 d62f63076585769f757dcaf9919d2f07fab113d3 - ivera - 2020-02-07 11:05:06] |
| 1> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) [randomizedtesting-runner-2.7.1.jar:?] |
| 1> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_241] |
| 1> Caused by: com.google.api.client.http.HttpResponseException: 410 Gone |
| 1> at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1094) ~[google-http-client-1.30.1.jar:?] |
| 1> at com.google.cloud.storage.spi.v1.HttpStorageRpc.write(HttpStorageRpc.java:739) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> ... 56 more |
| 1> Caused by: com.google.api.client.http.HttpResponseException: 410 Gone |
| 1> at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1094) ~[google-http-client-1.30.1.jar:?] |
| 1> at com.google.cloud.storage.spi.v1.HttpStorageRpc.write(HttpStorageRpc.java:739) ~[google-cloud-storage-1.77.0.jar:1.77.0] |
| 1> ... 58 more