From cbad1ca6e0ac3c7eade1c59f8364c55d41f9878d Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 14 Oct 2025 10:39:21 +0000 Subject: [PATCH 01/18] chore: Refactor retrier creation from HttpStorageOptions to StorageOptions --- .../com/google/cloud/storage/HttpStorageOptions.java | 11 +---------- .../java/com/google/cloud/storage/StorageOptions.java | 11 +++++++++++ .../java/com/example/storage/ITBucketSnippets.java | 1 + .../java/com/example/storage/ITObjectSnippets.java | 2 ++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java index b1400bcb62..a73b5b140a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java @@ -32,8 +32,6 @@ import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.spi.ServiceRpcFactory; import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory; -import com.google.cloud.storage.Retrying.DefaultRetrier; -import com.google.cloud.storage.Retrying.HttpRetrier; import com.google.cloud.storage.Retrying.RetryingDependencies; import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.TransportCompatibility.Transport; @@ -408,14 +406,7 @@ public Storage create(StorageOptions options) { } WriterFactory factory = blobWriteSessionConfig.createFactory(clock); StorageImpl storage = - new StorageImpl( - httpStorageOptions, - factory, - new HttpRetrier( - new DefaultRetrier( - OtelStorageDecorator.retryContextDecorator(otel), - RetryingDependencies.simple( - options.getClock(), options.getRetrySettings())))); + new StorageImpl(httpStorageOptions, factory, options.createRetrier()); return OtelStorageDecorator.decorate(storage, otel, Transport.HTTP); } catch (IOException e) { throw new IllegalStateException( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index 4dac2b43ef..27306b6d6f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -26,6 +26,10 @@ import com.google.cloud.storage.HttpStorageOptions.HttpStorageDefaults; import com.google.cloud.storage.HttpStorageOptions.HttpStorageFactory; import com.google.cloud.storage.HttpStorageOptions.HttpStorageRpcFactory; +import com.google.cloud.storage.Retrying.DefaultRetrier; +import com.google.cloud.storage.Retrying.HttpRetrier; +import com.google.cloud.storage.Retrying.Retrier; +import com.google.cloud.storage.Retrying.RetryingDependencies; import com.google.cloud.storage.Storage.BlobWriteOption; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.spi.StorageRpcFactory; @@ -68,6 +72,13 @@ public abstract class StorageOptions extends ServiceOptions Date: Tue, 21 Oct 2025 07:01:59 +0000 Subject: [PATCH 02/18] feat: Added API for CreateMultipartUpload. --- google-cloud-storage/pom.xml | 4 + .../cloud/storage/MultipartUploadClient.java | 43 +++ .../storage/MultipartUploadClientImpl.java | 150 ++++++++++ .../MultipartUploadHttpRequestManager.java | 51 ++++ .../storage/MultipartUploadSettings.java | 32 ++ .../google/cloud/storage/ObjectLockMode.java | 76 +++++ .../model/CreateMultipartUploadRequest.java | 280 ++++++++++++++++++ .../model/CreateMultipartUploadResponse.java | 163 ++++++++++ 8 files changed, 799 insertions(+) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 1b8cb3e8f1..855a1f999a 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -19,6 +19,10 @@ 1.123.5 + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + com.google.guava guava diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java new file mode 100644 index 0000000000..2675863a96 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -0,0 +1,43 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalExtensionOnly; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import java.io.IOException; +import java.net.URI; + +@BetaApi +@InternalExtensionOnly +public abstract class MultipartUploadClient { + + MultipartUploadClient() {} + + public abstract CreateMultipartUploadResponse createMultipartUpload( + CreateMultipartUploadRequest request) throws IOException; + + public static MultipartUploadClient create(MultipartUploadSettings config) { + HttpStorageOptions options = config.getOptions(); + return new MultipartUploadClientImpl( + URI.create(options.getHost()), + options.getStorageRpcV1().getStorage().getRequestFactory(), + options.createRetrier(), + options); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java new file mode 100644 index 0000000000..ece7d55025 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -0,0 +1,150 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.cloud.storage.Conversions.Decoder; +import com.google.cloud.storage.Retrying.Retrier; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.common.net.MediaType; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +public class MultipartUploadClientImpl extends MultipartUploadClient { + + private static final String GCS_ENDPOINT = "https://storage.googleapis.com"; + + private final MultipartUploadHttpRequestManager httpRequestManager; + private final XmlMapper xmlMapper; + private final HttpStorageOptions options; + private final Retrier retrier; + + public MultipartUploadClientImpl( + URI uri, HttpRequestFactory requestFactory, Retrier retrier, HttpStorageOptions options) { + this.httpRequestManager = new MultipartUploadHttpRequestManager(requestFactory); + this.xmlMapper = new XmlMapper(); + this.options = options; + this.retrier = retrier; + } + + private Map getGenericExtensionHeader() { + Map extensionHeaders = new HashMap<>(); + if (options.getClientLibToken() != null) { + extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); + } + if (options.getProjectId() != null) { + extensionHeaders.put("x-goog-user-project", options.getProjectId()); + } + extensionHeaders.put("Date", getRfc1123Date()); + return extensionHeaders; + } + + public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) + throws IOException { + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String uri = GCS_ENDPOINT + resourcePath + "?uploads"; + + String contentType; + if (request.getContentType() == null) { + contentType = "application/x-www-form-urlencoded"; + } else { + try { + contentType = MediaType.parse(request.getContentType()).toString(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid Content-Type header provided: " + request.getContentType(), e); + } + } + + HttpResponse response = + httpRequestManager.sendCreateMultipartUploadRequest( + uri, contentType, request, getExtensionHeadersForCreateMultipartUpload(request)); + + if (!response.isSuccessStatusCode()) { + String error = response.parseAsString(); + throw new RuntimeException( + "Failed to initiate upload: " + response.getStatusCode() + " " + error); + } + + return xmlMapper.readValue(response.getContent(), CreateMultipartUploadResponse.class); + } + + private Map getExtensionHeadersForCreateMultipartUpload( + CreateMultipartUploadRequest request) { + Map extensionHeaders = getGenericExtensionHeader(); + if (request.getCannedAcl() != null) { + extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); + } + if (request.getMetadata() != null) { + for (Map.Entry entry : request.getMetadata().entrySet()) { + if (entry.getKey() != null || entry.getValue() != null) { + extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); + } + } + } + if (request.getStorageClass() != null) { + extensionHeaders.put("x-goog-storage-class", request.getStorageClass().toString()); + } + if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { + extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); + } + // x-goog-object-lock-mode and x-goog-object-lock-retain-until-date should be specified together + // Refer: https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers + if (request.getObjectLockMode() != null && request.getObjectLockRetainUntilDate() != null) { + extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); + extensionHeaders.put( + "x-goog-object-lock-retain-until-date", + toRfc3339String(request.getObjectLockRetainUntilDate())); + } + if (request.getCustomTime() != null) { + extensionHeaders.put("x-goog-custom-time", toRfc3339String(request.getCustomTime())); + } + return extensionHeaders; + } + + private String encode(String value) throws UnsupportedEncodingException { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + } + + private String toRfc3339String(Date date) { + TimeZone tz = TimeZone.getTimeZone("UTC"); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + df.setTimeZone(tz); + return df.format(date); + } + + public static String getRfc1123Date() { + return DateTimeFormatter.RFC_1123_DATE_TIME + .withZone(ZoneId.of("GMT")) + .format(ZonedDateTime.now()); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java new file mode 100644 index 0000000000..44ed1b9816 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import com.google.api.client.http.ByteArrayContent; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import java.io.IOException; +import java.util.Map; + +public class MultipartUploadHttpRequestManager { + + private final HttpRequestFactory requestFactory; + + public MultipartUploadHttpRequestManager(HttpRequestFactory requestFactory) { + this.requestFactory = requestFactory; + } + + public HttpResponse sendCreateMultipartUploadRequest( + String uri, + String contentType, + CreateMultipartUploadRequest request, + Map extensionHeaders) + throws IOException { + HttpRequest httpRequest = + requestFactory.buildPostRequest( + new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); + httpRequest.getHeaders().setContentType(contentType); + for (Map.Entry entry : extensionHeaders.entrySet()) { + httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); + } + httpRequest.setThrowExceptionOnExecuteError(false); + return httpRequest.execute(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java new file mode 100644 index 0000000000..87596b3ebe --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java @@ -0,0 +1,32 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +public final class MultipartUploadSettings { + private final HttpStorageOptions options; + + private MultipartUploadSettings(HttpStorageOptions options) { + this.options = options; + } + + public HttpStorageOptions getOptions() { + return options; + } + + public static MultipartUploadSettings of(HttpStorageOptions options) { + return new MultipartUploadSettings(options); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java new file mode 100644 index 0000000000..c47582b97f --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java @@ -0,0 +1,76 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import com.google.api.core.ApiFunction; +import com.google.cloud.StringEnumType; +import com.google.cloud.StringEnumValue; + +/** + * Represents the object lock mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ +public final class ObjectLockMode extends StringEnumValue { + private static final long serialVersionUID = -1882734434792102329L; + + private ObjectLockMode(String constant) { + super(constant); + } + + private static final ApiFunction CONSTRUCTOR = + new ApiFunction() { + @Override + public ObjectLockMode apply(String constant) { + return new ObjectLockMode(constant); + } + }; + + private static final StringEnumType type = + new StringEnumType(ObjectLockMode.class, CONSTRUCTOR); + + /** + * Governance mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ + public static final ObjectLockMode GOVERNANCE = type.createAndRegister("GOVERNANCE"); + + /** + * Compliance mode. See https://cloud.google.com/storage/docs/object-lock + * for details. + */ + public static final ObjectLockMode COMPLIANCE = type.createAndRegister("COMPLIANCE"); + + /** + * Get the ObjectLockMode for the given String constant, and throw an exception if the constant is + * not recognized. + */ + public static ObjectLockMode valueOfStrict(String constant) { + return type.valueOfStrict(constant); + } + + /** Get the ObjectLockMode for the given String constant, and allow unrecognized values. */ + public static ObjectLockMode valueOf(String constant) { + return type.valueOf(constant); + } + + /** Return the known values for ObjectLockMode. */ + public static ObjectLockMode[] values() { + return type.values(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java new file mode 100644 index 0000000000..9ec1a89ccc --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -0,0 +1,280 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.google.cloud.storage.ObjectLockMode; +import com.google.cloud.storage.Storage.PredefinedAcl; +import com.google.cloud.storage.StorageClass; +import com.google.common.base.MoreObjects; +import java.util.Date; +import java.util.Map; +import java.util.Objects; + +public class CreateMultipartUploadRequest { + private final String bucket; + private final String key; + private final PredefinedAcl cannedAcl; + private final String contentType; + private final Map metadata; + private final StorageClass storageClass; + private final Date customTime; + private final String kmsKeyName; + private final ObjectLockMode objectLockMode; + private final Date objectLockRetainUntilDate; + + private CreateMultipartUploadRequest(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; + this.cannedAcl = builder.cannedAcl; + this.contentType = builder.contentType; + this.metadata = builder.metadata; + this.storageClass = builder.storageClass; + this.customTime = builder.customTime; + this.kmsKeyName = builder.kmsKeyName; + this.objectLockMode = builder.objectLockMode; + this.objectLockRetainUntilDate = builder.objectLockRetainUntilDate; + } + + public String bucket() { + return bucket; + } + + public String key() { + return key; + } + + public PredefinedAcl getCannedAcl() { + return cannedAcl; + } + + public String getContentType() { + return contentType; + } + + public Map getMetadata() { + return metadata; + } + + public StorageClass getStorageClass() { + return storageClass; + } + + public Date getCustomTime() { + return customTime; + } + + public String getKmsKeyName() { + return kmsKeyName; + } + + public ObjectLockMode getObjectLockMode() { + return objectLockMode; + } + + public Date getObjectLockRetainUntilDate() { + return objectLockRetainUntilDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateMultipartUploadRequest)) { + return false; + } + CreateMultipartUploadRequest that = (CreateMultipartUploadRequest) o; + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && cannedAcl == that.cannedAcl + && Objects.equals(contentType, that.contentType) + && Objects.equals(metadata, that.metadata) + && Objects.equals(storageClass, that.storageClass) + && Objects.equals(customTime, that.customTime) + && Objects.equals(kmsKeyName, that.kmsKeyName) + && objectLockMode == that.objectLockMode + && Objects.equals(objectLockRetainUntilDate, that.objectLockRetainUntilDate); + } + + @Override + public int hashCode() { + return Objects.hash( + bucket, + key, + cannedAcl, + contentType, + metadata, + storageClass, + customTime, + kmsKeyName, + objectLockMode, + objectLockRetainUntilDate); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("cannedAcl", cannedAcl) + .add("contentType", contentType) + .add("metadata", metadata) + .add("storageClass", storageClass) + .add("customTime", customTime) + .add("kmsKeyName", kmsKeyName) + .add("objectLockMode", objectLockMode) + .add("objectLockRetainUntilDate", objectLockRetainUntilDate) + .toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String bucket; + private String key; + private PredefinedAcl cannedAcl; + private String contentType; + private Map metadata; + private StorageClass storageClass; + private Date customTime; + private String kmsKeyName; + private ObjectLockMode objectLockMode; + private Date objectLockRetainUntilDate; + + private Builder() {} + + /** + * The bucket to which the object is being uploaded. + * + * @param bucket The bucket name + * @return this builder + */ + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + /** + * The name of the object. + * + * @param key The object name + * @return this builder + */ + public Builder key(String key) { + this.key = key; + return this; + } + + /** + * A canned ACL to apply to the object. + * + * @param cannedAcl The canned ACL + * @return this builder + */ + public Builder cannedAcl(PredefinedAcl cannedAcl) { + this.cannedAcl = cannedAcl; + return this; + } + + /** + * The MIME type of the data you are uploading. + * + * @param contentType The Content-Type + * @return this builder + */ + public Builder contentType(String contentType) { + this.contentType = contentType; + return this; + } + + /** + * The custom metadata of the object. + * + * @param metadata The custom metadata + * @return this builder + */ + public Builder metadata(Map metadata) { + this.metadata = metadata; + return this; + } + + /** + * Gives each part of the upload and the resulting object a storage class besides the default + * storage class of the associated bucket. + * + * @param storageClass The Storage-Class + * @return this builder + */ + public Builder storageClass(StorageClass storageClass) { + this.storageClass = storageClass; + return this; + } + + /** + * A user-specified date and time. + * + * @param customTime The custom time + * @return this builder + */ + public Builder customTime(Date customTime) { + this.customTime = customTime; + return this; + } + + /** + * The customer-managed encryption key to use to encrypt the object. Refer: Customer + * Managed Keys + * + * @param kmsKeyName The Cloud KMS key + * @return this builder + */ + public Builder kmsKeyName(String kmsKeyName) { + this.kmsKeyName = kmsKeyName; + return this; + } + + /** + * Mode of the object's retention configuration. GOVERNANCE corresponds to unlocked mode, and + * COMPLIANCE corresponds to locked mode. + * + * @param objectLockMode The object lock mode + * @return this builder + */ + public Builder objectLockMode(ObjectLockMode objectLockMode) { + this.objectLockMode = objectLockMode; + return this; + } + + /** + * Date that determines the time until which the object is retained as immutable. + * + * @param objectLockRetainUntilDate The object lock retention until date + * @return this builder + */ + public Builder objectLockRetainUntilDate(Date objectLockRetainUntilDate) { + this.objectLockRetainUntilDate = objectLockRetainUntilDate; + return this; + } + + public CreateMultipartUploadRequest build() { + return new CreateMultipartUploadRequest(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java new file mode 100644 index 0000000000..cd32bb1b99 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -0,0 +1,163 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.multipartupload.model; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.google.common.base.MoreObjects; +import java.util.Objects; + +/** + * Represents the response from a CreateMultipartUpload request. This class encapsulates the details + * of the initiated multipart upload, including the bucket, key, and the unique upload ID. + */ +@JacksonXmlRootElement(localName = "InitiateMultipartUploadResult") +public class CreateMultipartUploadResponse { + + @JacksonXmlProperty(localName = "Bucket") + private String bucket; + + @JacksonXmlProperty(localName = "Key") + private String key; + + @JacksonXmlProperty(localName = "UploadId") + private String uploadId; + + private CreateMultipartUploadResponse(Builder builder) { + this.bucket = builder.bucket; + this.key = builder.key; + this.uploadId = builder.uploadId; + } + + private CreateMultipartUploadResponse() {} + + /** + * Returns the name of the bucket where the multipart upload was initiated. + * + * @return The bucket name. + */ + public String bucket() { + return bucket; + } + + /** + * Returns the key (object name) for which the multipart upload was initiated. + * + * @return The object key. + */ + public String key() { + return key; + } + + /** + * Returns the unique identifier for this multipart upload. This ID must be included in all + * subsequent requests related to this upload (e.g., uploading parts, completing the upload). + * + * @return The upload ID. + */ + public String uploadId() { + return uploadId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CreateMultipartUploadResponse)) { + return false; + } + CreateMultipartUploadResponse that = (CreateMultipartUploadResponse) o; + return Objects.equals(bucket, that.bucket) + && Objects.equals(key, that.key) + && Objects.equals(uploadId, that.uploadId); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, key, uploadId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("bucket", bucket) + .add("key", key) + .add("uploadId", uploadId) + .toString(); + } + + /** + * Creates a new builder for {@link CreateMultipartUploadResponse}. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** A builder for {@link CreateMultipartUploadResponse} objects. */ + public static class Builder { + private String bucket; + private String key; + private String uploadId; + + private Builder() {} + + /** + * Sets the bucket name for the multipart upload. + * + * @param bucket The bucket name. + * @return This builder. + */ + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + /** + * Sets the key (object name) for the multipart upload. + * + * @param key The object key. + * @return This builder. + */ + public Builder key(String key) { + this.key = key; + return this; + } + + /** + * Sets the upload ID for the multipart upload. + * + * @param uploadId The upload ID. + * @return This builder. + */ + public Builder uploadId(String uploadId) { + this.uploadId = uploadId; + return this; + } + + /** + * Builds a new {@link CreateMultipartUploadResponse} object. + * + * @return A new {@link CreateMultipartUploadResponse} object. + */ + public CreateMultipartUploadResponse build() { + return new CreateMultipartUploadResponse(this); + } + } +} From 3bfd51b93dbb8feed654e97598088b0da835c108 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 21 Oct 2025 18:41:14 +0000 Subject: [PATCH 03/18] added missing javadoc --- .../multipartupload/model/CreateMultipartUploadRequest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 8398182ce2..b52eeebb61 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -24,6 +24,10 @@ import java.util.Map; import java.util.Objects; +/** + * Represents a request to initiate a multipart upload. + * This class holds all the necessary information to create a new multipart upload session. + */ public final class CreateMultipartUploadRequest { private final String bucket; private final String key; From 11570046edb2b0b36ca157eb488c8ea13bbd5f42 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Tue, 21 Oct 2025 18:47:17 +0000 Subject: [PATCH 04/18] Fixed lint issues --- .../multipartupload/model/CreateMultipartUploadRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index b52eeebb61..0cd121a9b4 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -25,8 +25,8 @@ import java.util.Objects; /** - * Represents a request to initiate a multipart upload. - * This class holds all the necessary information to create a new multipart upload session. + * Represents a request to initiate a multipart upload. This class holds all the necessary + * information to create a new multipart upload session. */ public final class CreateMultipartUploadRequest { private final String bucket; From 585707d1d1fde5d6a573166486f6cde8905590db Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 09:06:49 +0000 Subject: [PATCH 05/18] chore: adding review changes. --- .../cloud/storage/MultipartUploadClient.java | 2 + .../storage/MultipartUploadClientImpl.java | 14 ++-- .../storage/MultipartUploadSettings.java | 5 ++ .../google/cloud/storage/ObjectLockMode.java | 2 + .../model/CreateMultipartUploadRequest.java | 82 ++++++++++++++++--- .../model/CreateMultipartUploadResponse.java | 4 +- 6 files changed, 90 insertions(+), 19 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index 6a0d28112c..0d1a560034 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -37,9 +37,11 @@ public abstract class MultipartUploadClient { MultipartUploadClient() {} + @BetaApi public abstract CreateMultipartUploadResponse createMultipartUpload( CreateMultipartUploadRequest request) throws IOException; + @BetaApi public static MultipartUploadClient create(MultipartUploadSettings config) { HttpStorageOptions options = config.getOptions(); return new MultipartUploadClientImpl( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 6d070c9e54..fa3ac2b744 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -16,6 +16,7 @@ package com.google.cloud.storage; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.core.BetaApi; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.cloud.storage.Retrying.Retrier; @@ -27,19 +28,18 @@ import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; +import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.TimeZone; /** * This class is an implementation of {@link MultipartUploadClient} that uses the Google Cloud * Storage XML API to perform multipart uploads. */ +@BetaApi public final class MultipartUploadClientImpl extends MultipartUploadClient { private final MultipartUploadHttpRequestManager httpRequestManager; @@ -69,6 +69,7 @@ private Map getGenericExtensionHeader() { return extensionHeaders; } + @BetaApi public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) throws IOException { String encodedBucket = encode(request.bucket()); @@ -138,11 +139,8 @@ private String encode(String value) throws UnsupportedEncodingException { return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } - private String toRfc3339String(Date date) { - TimeZone tz = TimeZone.getTimeZone("UTC"); - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - df.setTimeZone(tz); - return df.format(date); + private String toRfc3339String(OffsetDateTime dateTime) { + return DateTimeFormatter.ISO_INSTANT.format(dateTime); } public static String getRfc1123Date() { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java index 596020e8aa..fbf55b3bfd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadSettings.java @@ -15,11 +15,14 @@ */ package com.google.cloud.storage; +import com.google.api.core.BetaApi; + /** * Settings for configuring the {@link MultipartUploadClient}. * *

This class is for internal use only and is not intended for public consumption. */ +@BetaApi public final class MultipartUploadSettings { private final HttpStorageOptions options; @@ -37,6 +40,7 @@ private MultipartUploadSettings(HttpStorageOptions options) { * * @return The {@link HttpStorageOptions}. */ + @BetaApi public HttpStorageOptions getOptions() { return options; } @@ -48,6 +52,7 @@ public HttpStorageOptions getOptions() { * @param options The {@link HttpStorageOptions} to use. * @return A new {@code MultipartUploadSettings} instance. */ + @BetaApi public static MultipartUploadSettings of(HttpStorageOptions options) { return new MultipartUploadSettings(options); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java index c47582b97f..798df8db9b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java @@ -18,12 +18,14 @@ import com.google.api.core.ApiFunction; import com.google.cloud.StringEnumType; import com.google.cloud.StringEnumValue; +import com.google.api.core.BetaApi; /** * Represents the object lock mode. See https://cloud.google.com/storage/docs/object-lock * for details. */ +@BetaApi public final class ObjectLockMode extends StringEnumValue { private static final long serialVersionUID = -1882734434792102329L; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 0cd121a9b4..ce33c90249 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -16,11 +16,12 @@ package com.google.cloud.storage.multipartupload.model; +import com.google.api.core.BetaApi; import com.google.cloud.storage.ObjectLockMode; import com.google.cloud.storage.Storage.PredefinedAcl; import com.google.cloud.storage.StorageClass; import com.google.common.base.MoreObjects; -import java.util.Date; +import java.time.OffsetDateTime; import java.util.Map; import java.util.Objects; @@ -28,6 +29,7 @@ * Represents a request to initiate a multipart upload. This class holds all the necessary * information to create a new multipart upload session. */ +@BetaApi public final class CreateMultipartUploadRequest { private final String bucket; private final String key; @@ -35,10 +37,10 @@ public final class CreateMultipartUploadRequest { private final String contentType; private final Map metadata; private final StorageClass storageClass; - private final Date customTime; + private final OffsetDateTime customTime; private final String kmsKeyName; private final ObjectLockMode objectLockMode; - private final Date objectLockRetainUntilDate; + private final OffsetDateTime objectLockRetainUntilDate; private CreateMultipartUploadRequest(Builder builder) { this.bucket = builder.bucket; @@ -53,43 +55,93 @@ private CreateMultipartUploadRequest(Builder builder) { this.objectLockRetainUntilDate = builder.objectLockRetainUntilDate; } + /** + * Returns the name of the bucket to which the object is being uploaded. + * + * @return The bucket name + */ public String bucket() { return bucket; } + /** + * Returns the name of the object. + * + * @return The object name + */ public String key() { return key; } + /** + * Returns a canned ACL to apply to the object. + * + * @return The canned ACL + */ public PredefinedAcl getCannedAcl() { return cannedAcl; } + /** + * Returns the MIME type of the data you are uploading. + * + * @return The Content-Type + */ public String getContentType() { return contentType; } + /** + * Returns the custom metadata of the object. + * + * @return The custom metadata + */ public Map getMetadata() { return metadata; } + /** + * Returns the storage class for the object. + * + * @return The Storage-Class + */ public StorageClass getStorageClass() { return storageClass; } - public Date getCustomTime() { + /** + * Returns a user-specified date and time. + * + * @return The custom time + */ + public OffsetDateTime getCustomTime() { return customTime; } + /** + * Returns the customer-managed encryption key to use to encrypt the object. + * + * @return The Cloud KMS key + */ public String getKmsKeyName() { return kmsKeyName; } + /** + * Returns the mode of the object's retention configuration. + * + * @return The object lock mode + */ public ObjectLockMode getObjectLockMode() { return objectLockMode; } - public Date getObjectLockRetainUntilDate() { + /** + * Returns the date that determines the time until which the object is retained as immutable. + * + * @return The object lock retention until date + */ + public OffsetDateTime getObjectLockRetainUntilDate() { return objectLockRetainUntilDate; } @@ -145,21 +197,26 @@ public String toString() { .toString(); } + /** + * Returns a new {@link Builder} for creating a {@link CreateMultipartUploadRequest}. + * + * @return a new builder + */ public static Builder builder() { return new Builder(); } - public static class Builder { + public static final class Builder { private String bucket; private String key; private PredefinedAcl cannedAcl; private String contentType; private Map metadata; private StorageClass storageClass; - private Date customTime; + private OffsetDateTime customTime; private String kmsKeyName; private ObjectLockMode objectLockMode; - private Date objectLockRetainUntilDate; + private OffsetDateTime objectLockRetainUntilDate; private Builder() {} @@ -236,7 +293,7 @@ public Builder storageClass(StorageClass storageClass) { * @param customTime The custom time * @return this builder */ - public Builder customTime(Date customTime) { + public Builder customTime(OffsetDateTime customTime) { this.customTime = customTime; return this; } @@ -272,11 +329,16 @@ public Builder objectLockMode(ObjectLockMode objectLockMode) { * @param objectLockRetainUntilDate The object lock retention until date * @return this builder */ - public Builder objectLockRetainUntilDate(Date objectLockRetainUntilDate) { + public Builder objectLockRetainUntilDate(OffsetDateTime objectLockRetainUntilDate) { this.objectLockRetainUntilDate = objectLockRetainUntilDate; return this; } + /** + * Creates a new {@link CreateMultipartUploadRequest} object. + * + * @return a new {@link CreateMultipartUploadRequest} object + */ public CreateMultipartUploadRequest build() { return new CreateMultipartUploadRequest(this); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java index a29a7ee2b9..46f86791a8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -16,6 +16,7 @@ package com.google.cloud.storage.multipartupload.model; +import com.google.api.core.BetaApi; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import com.google.common.base.MoreObjects; @@ -26,6 +27,7 @@ * of the initiated multipart upload, including the bucket, key, and the unique upload ID. */ @JacksonXmlRootElement(localName = "InitiateMultipartUploadResult") +@BetaApi public final class CreateMultipartUploadResponse { @JacksonXmlProperty(localName = "Bucket") @@ -111,7 +113,7 @@ public static Builder builder() { } /** A builder for {@link CreateMultipartUploadResponse} objects. */ - public static class Builder { + public static final class Builder { private String bucket; private String key; private String uploadId; From eca5d549c6c61268f991c297c2e1fbc2ca35c351 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 14:55:59 +0000 Subject: [PATCH 06/18] chore: addind review comment fixes --- .../storage/MultipartUploadClientImpl.java | 110 +--------- .../MultipartUploadHttpRequestManager.java | 93 +++++++-- .../google/cloud/storage/ObjectLockMode.java | 2 +- .../google/cloud/storage/XmlObjectParser.java | 53 +++++ .../model/CreateMultipartUploadResponse.java | 2 +- .../MultipartUploadClientImplTest.java | 192 ------------------ ...MultipartUploadHttpRequestManagerTest.java | 69 ------- 7 files changed, 142 insertions(+), 379 deletions(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java delete mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java delete mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadHttpRequestManagerTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index fa3ac2b744..8c333455a7 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -16,136 +16,38 @@ package com.google.cloud.storage; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.api.core.BetaApi; import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpResponse; +import com.google.api.core.BetaApi; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; -import com.google.common.net.MediaType; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.net.URI; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; -import java.util.Map; /** * This class is an implementation of {@link MultipartUploadClient} that uses the Google Cloud * Storage XML API to perform multipart uploads. */ @BetaApi -public final class MultipartUploadClientImpl extends MultipartUploadClient { +final class MultipartUploadClientImpl extends MultipartUploadClient { private final MultipartUploadHttpRequestManager httpRequestManager; - private final XmlMapper xmlMapper; private final HttpStorageOptions options; private final Retrier retrier; private final URI uri; - public MultipartUploadClientImpl( + MultipartUploadClientImpl( URI uri, HttpRequestFactory requestFactory, Retrier retrier, HttpStorageOptions options) { - this.httpRequestManager = new MultipartUploadHttpRequestManager(requestFactory); - this.xmlMapper = new XmlMapper(); + this.httpRequestManager = + new MultipartUploadHttpRequestManager(requestFactory, new XmlObjectParser(new XmlMapper())); this.options = options; this.retrier = retrier; this.uri = uri; } - private Map getGenericExtensionHeader() { - Map extensionHeaders = new HashMap<>(); - if (options.getClientLibToken() != null) { - extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); - } - if (options.getProjectId() != null) { - extensionHeaders.put("x-goog-user-project", options.getProjectId()); - } - extensionHeaders.put("Date", getRfc1123Date()); - return extensionHeaders; - } - @BetaApi public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) throws IOException { - String encodedBucket = encode(request.bucket()); - String encodedKey = encode(request.key()); - String resourcePath = "/" + encodedBucket + "/" + encodedKey; - String createUri = uri.toString() + resourcePath + "?uploads"; - - String contentType; - if (request.getContentType() == null) { - contentType = "application/x-www-form-urlencoded"; - } else { - try { - contentType = MediaType.parse(request.getContentType()).toString(); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid Content-Type header provided: " + request.getContentType(), e); - } - } - - HttpResponse response = - httpRequestManager.sendCreateMultipartUploadRequest( - createUri, contentType, request, getExtensionHeadersForCreateMultipartUpload(request)); - - if (!response.isSuccessStatusCode()) { - String error = response.parseAsString(); - throw new RuntimeException( - "Failed to initiate upload: " + response.getStatusCode() + " " + error); - } - - return xmlMapper.readValue(response.getContent(), CreateMultipartUploadResponse.class); - } - - private Map getExtensionHeadersForCreateMultipartUpload( - CreateMultipartUploadRequest request) { - Map extensionHeaders = getGenericExtensionHeader(); - if (request.getCannedAcl() != null) { - extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); - } - if (request.getMetadata() != null) { - for (Map.Entry entry : request.getMetadata().entrySet()) { - if (entry.getKey() != null || entry.getValue() != null) { - extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); - } - } - } - if (request.getStorageClass() != null) { - extensionHeaders.put("x-goog-storage-class", request.getStorageClass().toString()); - } - if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { - extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); - } - // x-goog-object-lock-mode and x-goog-object-lock-retain-until-date should be specified together - // Refer: https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers - if (request.getObjectLockMode() != null && request.getObjectLockRetainUntilDate() != null) { - extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); - extensionHeaders.put( - "x-goog-object-lock-retain-until-date", - toRfc3339String(request.getObjectLockRetainUntilDate())); - } - if (request.getCustomTime() != null) { - extensionHeaders.put("x-goog-custom-time", toRfc3339String(request.getCustomTime())); - } - return extensionHeaders; - } - - private String encode(String value) throws UnsupportedEncodingException { - return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); - } - - private String toRfc3339String(OffsetDateTime dateTime) { - return DateTimeFormatter.ISO_INSTANT.format(dateTime); - } - - public static String getRfc1123Date() { - return DateTimeFormatter.RFC_1123_DATE_TIME - .withZone(ZoneId.of("GMT")) - .format(ZonedDateTime.now()); + return httpRequestManager.sendCreateMultipartUploadRequest(uri, request, options); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java index 27137852c6..809861db2a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java @@ -19,33 +19,102 @@ import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpResponse; +import com.google.api.client.util.ObjectParser; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; import java.util.Map; final class MultipartUploadHttpRequestManager { private final HttpRequestFactory requestFactory; + private final ObjectParser objectParser; - MultipartUploadHttpRequestManager(HttpRequestFactory requestFactory) { + MultipartUploadHttpRequestManager(HttpRequestFactory requestFactory, ObjectParser objectParser) { this.requestFactory = requestFactory; + this.objectParser = objectParser; } - HttpResponse sendCreateMultipartUploadRequest( - String uri, - String contentType, - CreateMultipartUploadRequest request, - Map extensionHeaders) + CreateMultipartUploadResponse sendCreateMultipartUploadRequest( + URI uri, CreateMultipartUploadRequest request, HttpStorageOptions options) throws IOException { + + String encodedBucket = encode(request.bucket()); + String encodedKey = encode(request.key()); + String resourcePath = "/" + encodedBucket + "/" + encodedKey; + String createUri = uri.toString() + resourcePath + "?uploads"; + HttpRequest httpRequest = requestFactory.buildPostRequest( - new GenericUrl(uri), new ByteArrayContent(contentType, new byte[0])); - httpRequest.getHeaders().setContentType(contentType); - for (Map.Entry entry : extensionHeaders.entrySet()) { + new GenericUrl(createUri), + new ByteArrayContent(request.getContentType(), new byte[0])); + httpRequest.getHeaders().setContentType(request.getContentType()); + for (Map.Entry entry : + getExtensionHeadersForCreateMultipartUpload(request, options).entrySet()) { httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); } - httpRequest.setThrowExceptionOnExecuteError(false); - return httpRequest.execute(); + httpRequest.setParser(objectParser); + httpRequest.setThrowExceptionOnExecuteError(true); + return httpRequest.execute().parseAs(CreateMultipartUploadResponse.class); + } + + private Map getExtensionHeadersForCreateMultipartUpload( + CreateMultipartUploadRequest request, HttpStorageOptions options) { + Map extensionHeaders = getGenericExtensionHeader(options); + if (request.getCannedAcl() != null) { + extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); + } + //TODO(shreyassinha) Add encoding for x-goog-meta-* headers + if (request.getMetadata() != null) { + for (Map.Entry entry : request.getMetadata().entrySet()) { + if (entry.getKey() != null || entry.getValue() != null) { + extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); + } + } + } + if (request.getStorageClass() != null) { + extensionHeaders.put("x-goog-storage-class", request.getStorageClass().toString()); + } + if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { + extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); + } + if (request.getObjectLockMode() != null) { + extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); + } + if (request.getObjectLockRetainUntilDate() != null) { + extensionHeaders.put( + "x-goog-object-lock-retain-until-date", + toRfc3339String(request.getObjectLockRetainUntilDate())); + } + if (request.getCustomTime() != null) { + extensionHeaders.put("x-goog-custom-time", toRfc3339String(request.getCustomTime())); + } + return extensionHeaders; + } + + private Map getGenericExtensionHeader(HttpStorageOptions options) { + Map extensionHeaders = new HashMap<>(); + if (options.getClientLibToken() != null) { + extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); + } + if (options.getProjectId() != null) { + extensionHeaders.put("x-goog-user-project", options.getProjectId()); + } + return extensionHeaders; + } + + private String encode(String value) throws UnsupportedEncodingException { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + } + + private String toRfc3339String(OffsetDateTime dateTime) { + return DateTimeFormatter.ISO_INSTANT.format(dateTime); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java index 798df8db9b..f08ec2e08f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java @@ -16,9 +16,9 @@ package com.google.cloud.storage; import com.google.api.core.ApiFunction; +import com.google.api.core.BetaApi; import com.google.cloud.StringEnumType; import com.google.cloud.StringEnumValue; -import com.google.api.core.BetaApi; /** * Represents the object lock mode. See T parseAndClose(InputStream in, Charset charset, Class dataClass) + throws IOException { + return xmlMapper.readValue(in, dataClass); + } + + @Override + public Object parseAndClose(InputStream in, Charset charset, Type dataType) throws IOException { + return xmlMapper.readValue(in, xmlMapper.getTypeFactory().constructType(dataType)); + } + + @Override + public T parseAndClose(Reader reader, Class dataClass) throws IOException { + return xmlMapper.readValue(reader, dataClass); + } + + @Override + public Object parseAndClose(Reader reader, Type dataType) throws IOException { + return xmlMapper.readValue(reader, xmlMapper.getTypeFactory().constructType(dataType)); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java index 46f86791a8..35d2b2c171 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -16,9 +16,9 @@ package com.google.cloud.storage.multipartupload.model; -import com.google.api.core.BetaApi; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import com.google.api.core.BetaApi; import com.google.common.base.MoreObjects; import java.util.Objects; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java deleted file mode 100644 index 8408b93cc7..0000000000 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadClientImplTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.storage; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.http.LowLevelHttpResponse; -import com.google.api.client.json.Json; -import com.google.api.client.testing.http.MockHttpTransport; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.api.client.testing.http.MockLowLevelHttpResponse; -import com.google.cloud.storage.Retrying.Retrier; -import com.google.cloud.storage.Storage.PredefinedAcl; -import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; -import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; -import java.io.IOException; -import java.net.URI; -import java.util.Collections; -import java.util.Map; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -@RunWith(JUnit4.class) -public class MultipartUploadClientImplTest { - - private static final String BUCKET = "bucket"; - private static final String KEY = "key"; - private static final String UPLOAD_ID = "uploadId"; - private static final String CONTENT_TYPE = "application/octet-stream"; - private static final String PROJECT_ID = "project-id"; - - private MultipartUploadClientImpl multipartUploadClient; - - @Mock private MultipartUploadHttpRequestManager httpRequestManager; - - @Mock private Retrier retrier; - - @Captor private ArgumentCaptor> extensionHeadersCaptor; - - @Captor private ArgumentCaptor uriCaptor; - - private final XmlMapper xmlMapper = new XmlMapper(); - private AutoCloseable mocks; - - @Before - public void setUp() throws Exception { - mocks = MockitoAnnotations.openMocks(this); - HttpStorageOptions options = HttpStorageOptions.newBuilder().setProjectId(PROJECT_ID).build(); - multipartUploadClient = - new MultipartUploadClientImpl( - new URI("https://storage.googleapis.com"), null, retrier, options); - // Replace the httpRequestManager with a mock - java.lang.reflect.Field field = - MultipartUploadClientImpl.class.getDeclaredField("httpRequestManager"); - field.setAccessible(true); - field.set(multipartUploadClient, httpRequestManager); - } - - @After - public void tearDown() throws Exception { - mocks.close(); - } - - @Test - public void testCreateMultipartUpload() throws IOException { - CreateMultipartUploadRequest request = - CreateMultipartUploadRequest.builder() - .bucket(BUCKET) - .key(KEY) - .contentType(CONTENT_TYPE) - .build(); - CreateMultipartUploadResponse expectedResponse = - CreateMultipartUploadResponse.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); - String responseXml = xmlMapper.writeValueAsString(expectedResponse); - HttpResponse httpResponse = createHttpResponse(200, responseXml); - - when(httpRequestManager.sendCreateMultipartUploadRequest( - any(String.class), any(String.class), any(), any())) - .thenReturn(httpResponse); - - CreateMultipartUploadResponse actualResponse = - multipartUploadClient.createMultipartUpload(request); - - assertThat(actualResponse.bucket()).isEqualTo(expectedResponse.bucket()); - assertThat(actualResponse.key()).isEqualTo(expectedResponse.key()); - assertThat(actualResponse.uploadId()).isEqualTo(expectedResponse.uploadId()); - } - - @Test - public void testCreateMultipartUpload_withHeaders() throws IOException { - Map metadata = Collections.singletonMap("key", "value"); - CreateMultipartUploadRequest request = - CreateMultipartUploadRequest.builder() - .bucket(BUCKET) - .key(KEY) - .contentType(CONTENT_TYPE) - .cannedAcl(PredefinedAcl.PRIVATE) - .metadata(metadata) - .storageClass(StorageClass.COLDLINE) - .build(); - CreateMultipartUploadResponse expectedResponse = - CreateMultipartUploadResponse.builder().bucket(BUCKET).key(KEY).uploadId(UPLOAD_ID).build(); - String responseXml = xmlMapper.writeValueAsString(expectedResponse); - HttpResponse httpResponse = createHttpResponse(200, responseXml); - - when(httpRequestManager.sendCreateMultipartUploadRequest( - any(String.class), any(String.class), any(), extensionHeadersCaptor.capture())) - .thenReturn(httpResponse); - - multipartUploadClient.createMultipartUpload(request); - - Map capturedHeaders = extensionHeadersCaptor.getValue(); - assertThat(capturedHeaders).containsEntry("x-goog-acl", "PRIVATE"); - assertThat(capturedHeaders).containsEntry("x-goog-meta-key", "value"); - assertThat(capturedHeaders).containsEntry("x-goog-storage-class", "COLDLINE"); - } - - @Test - public void testCreateMultipartUpload_error() throws IOException { - CreateMultipartUploadRequest request = - CreateMultipartUploadRequest.builder() - .bucket(BUCKET) - .key(KEY) - .contentType(CONTENT_TYPE) - .build(); - HttpResponse httpResponse = createHttpResponse(500, "Internal Server Error"); - - when(httpRequestManager.sendCreateMultipartUploadRequest( - any(String.class), any(String.class), any(), any())) - .thenReturn(httpResponse); - - RuntimeException exception = - assertThrows( - RuntimeException.class, () -> multipartUploadClient.createMultipartUpload(request)); - assertThat(exception.getMessage()) - .isEqualTo("Failed to initiate upload: 500 Internal Server Error"); - } - - private HttpResponse createHttpResponse(int statusCode, String content) throws IOException { - HttpTransport transport = - new MockHttpTransport() { - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - return new MockLowLevelHttpRequest() { - @Override - public LowLevelHttpResponse execute() { - MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); - response.setStatusCode(statusCode); - response.setContentType(Json.MEDIA_TYPE); - response.setContent(content); - return response; - } - }; - } - }; - HttpRequestFactory requestFactory = transport.createRequestFactory(); - HttpRequest request = - requestFactory.buildGetRequest(new GenericUrl("https://storage.googleapis.com")); - request.setThrowExceptionOnExecuteError(false); - return request.execute(); - } -} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadHttpRequestManagerTest.java deleted file mode 100644 index 4f12c2badb..0000000000 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/MultipartUploadHttpRequestManagerTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.testing.http.MockHttpTransport; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class MultipartUploadHttpRequestManagerTest { - - private static final String BUCKET = "bucket"; - private static final String KEY = "key"; - private static final String CONTENT_TYPE = "application/octet-stream"; - private static final String URI = "https://storage.googleapis.com/" + BUCKET + "/" + KEY; - - @Test - public void testSendCreateMultipartUploadRequest() throws IOException { - final AtomicReference capturedMethod = new AtomicReference<>(); - final MockLowLevelHttpRequest lowLevelRequest = new MockLowLevelHttpRequest(); - HttpTransport transport = - new MockHttpTransport() { - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - capturedMethod.set(method); - lowLevelRequest.setUrl(url); - return lowLevelRequest; - } - }; - MultipartUploadHttpRequestManager httpRequestManager = - new MultipartUploadHttpRequestManager(transport.createRequestFactory()); - - CreateMultipartUploadRequest createRequest = - CreateMultipartUploadRequest.builder().bucket(BUCKET).key(KEY).build(); - Map headers = Collections.singletonMap("x-goog-test-header", "test-value"); - - httpRequestManager.sendCreateMultipartUploadRequest(URI, CONTENT_TYPE, createRequest, headers); - - assertThat(capturedMethod.get()).isEqualTo("POST"); - assertThat(lowLevelRequest.getUrl()).isEqualTo(URI); - assertThat(lowLevelRequest.getHeaderValues("x-goog-test-header")).containsExactly("test-value"); - assertThat(lowLevelRequest.getContentAsString()).isEqualTo(""); - } -} From 04ff3d5d6d4f2fe8ca0c8f04ca06310416dbdc1f Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 16:18:32 +0000 Subject: [PATCH 07/18] chore: added test cases --- ...MultipartUploadHttpRequestManagerTest.java | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java new file mode 100644 index 0000000000..cceb908fc5 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java @@ -0,0 +1,375 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; +import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus.OK; +import static org.junit.Assert.assertThrows; + +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonObjectParser; +import com.google.api.client.json.gson.GsonFactory; +import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.ParallelFriendly; +import com.google.cloud.storage.it.runner.annotations.SingleBackend; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; +import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.common.collect.ImmutableMap; +import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; +import io.grpc.netty.shaded.io.netty.buffer.Unpooled; +import io.grpc.netty.shaded.io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.grpc.netty.shaded.io.netty.handler.codec.http.FullHttpResponse; +import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus; +import java.net.URI; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@SingleBackend(Backend.PROD) +@ParallelFriendly +public final class ITMultipartUploadHttpRequestManagerTest { + private static final GsonFactory gson = GsonFactory.getDefaultInstance(); + private static final NetHttpTransport transport = new NetHttpTransport.Builder().build(); + private MultipartUploadHttpRequestManager multipartUploadHttpRequestManager; + private HttpStorageOptions httpStorageOptions; + + @Rule public final TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void setUp() throws Exception { + multipartUploadHttpRequestManager = + new MultipartUploadHttpRequestManager( + transport.createRequestFactory(), new JsonObjectParser(gson)); + httpStorageOptions = HttpStorageOptions.newBuilder().setProjectId("test-project").build(); + } + + @Test + public void sendCreateMultipartUploadRequest_success() throws Exception { + HttpRequestHandler handler = + req -> { + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .build(); + + CreateMultipartUploadResponse response = + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + + assertThat(response).isNotNull(); + assertThat(response.bucket()).isEqualTo("test-bucket"); + assertThat(response.key()).isEqualTo("test-key"); + assertThat(response.uploadId()).isEqualTo("test-upload-id"); + } + } + + @Test + public void sendCreateMultipartUploadRequest_error() throws Exception { + HttpRequestHandler handler = + req -> { + FullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), HttpResponseStatus.BAD_REQUEST); + resp.headers().set(CONTENT_TYPE, "text/plain; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .build(); + + StorageException se = + assertThrows( + StorageException.class, + () -> + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions)); + assertThat(se.getCode()).isEqualTo(400); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withCannedAcl() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-acl")).isEqualTo("authenticatedRead"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .cannedAcl(Storage.PredefinedAcl.AUTHENTICATED_READ) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withMetadata() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-meta-key1")).isEqualTo("value1"); + assertThat(req.headers().get("x-goog-meta-key2")).isEqualTo("value2"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .metadata(ImmutableMap.of("key1", "value1", "key2", "value2")) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withStorageClass() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-storage-class")).isEqualTo("ARCHIVE"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .storageClass(StorageClass.ARCHIVE) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withKmsKeyName() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-encryption-kms-key-name")) + .isEqualTo("projects/p/locations/l/keyRings/r/cryptoKeys/k"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .kmsKeyName("projects/p/locations/l/keyRings/r/cryptoKeys/k") + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withObjectLockMode() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-object-lock-mode")).isEqualTo("GOVERNANCE"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .objectLockMode(ObjectLockMode.GOVERNANCE) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withObjectLockRetainUntilDate() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-object-lock-retain-until-date")) + .isEqualTo("2024-01-01T00:00:00Z"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .objectLockRetainUntilDate(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } + + @Test + public void sendCreateMultipartUploadRequest_withCustomTime() throws Exception { + HttpRequestHandler handler = + req -> { + assertThat(req.headers().get("x-goog-custom-time")).isEqualTo("2024-01-01T00:00:00Z"); + CreateMultipartUploadResponse response = + CreateMultipartUploadResponse.builder() + .bucket("test-bucket") + .key("test-key") + .uploadId("test-upload-id") + .build(); + ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + + DefaultFullHttpResponse resp = + new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); + resp.headers().set(CONTENT_TYPE, "application/xml; charset=utf-8"); + return resp; + }; + + try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) { + URI endpoint = fakeHttpServer.getEndpoint(); + CreateMultipartUploadRequest request = + CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .contentType("application/octet-stream") + .customTime(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)) + .build(); + + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions); + } + } +} From f4e30f9ae65e5f59d4eceb0507de843c45d8be8f Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 16:52:17 +0000 Subject: [PATCH 08/18] chore: fixed IT tests --- .../cloud/storage/MultipartUploadHttpRequestManager.java | 5 ++--- .../main/java/com/google/cloud/storage/XmlObjectParser.java | 2 ++ .../multipartupload/model/CreateMultipartUploadRequest.java | 2 ++ .../storage/ITMultipartUploadHttpRequestManagerTest.java | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java index 809861db2a..78e5b19520 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java @@ -53,8 +53,7 @@ CreateMultipartUploadResponse sendCreateMultipartUploadRequest( HttpRequest httpRequest = requestFactory.buildPostRequest( - new GenericUrl(createUri), - new ByteArrayContent(request.getContentType(), new byte[0])); + new GenericUrl(createUri), new ByteArrayContent(request.getContentType(), new byte[0])); httpRequest.getHeaders().setContentType(request.getContentType()); for (Map.Entry entry : getExtensionHeadersForCreateMultipartUpload(request, options).entrySet()) { @@ -71,7 +70,7 @@ private Map getExtensionHeadersForCreateMultipartUpload( if (request.getCannedAcl() != null) { extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); } - //TODO(shreyassinha) Add encoding for x-goog-meta-* headers + // TODO(shreyassinha) Add encoding for x-goog-meta-* headers if (request.getMetadata() != null) { for (Map.Entry entry : request.getMetadata().entrySet()) { if (entry.getKey() != null || entry.getValue() != null) { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java index 5ab9712141..6893f53944 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.util.ObjectParser; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -26,6 +27,7 @@ final class XmlObjectParser implements ObjectParser { private final XmlMapper xmlMapper; + @VisibleForTesting public XmlObjectParser(XmlMapper xmlMapper) { this.xmlMapper = xmlMapper; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index ce33c90249..722b140c24 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -67,6 +67,7 @@ public String bucket() { /** * Returns the name of the object. * + * @see Object Naming * @return The object name */ public String key() { @@ -234,6 +235,7 @@ public Builder bucket(String bucket) { /** * The name of the object. * + * @see Object Naming * @param key The object name * @return this builder */ diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java index cceb908fc5..ff8ba131a0 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java @@ -21,8 +21,8 @@ import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus.OK; import static org.junit.Assert.assertThrows; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonObjectParser; import com.google.api.client.json.gson.GsonFactory; import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler; import com.google.cloud.storage.it.runner.StorageITRunner; @@ -61,7 +61,7 @@ public final class ITMultipartUploadHttpRequestManagerTest { public void setUp() throws Exception { multipartUploadHttpRequestManager = new MultipartUploadHttpRequestManager( - transport.createRequestFactory(), new JsonObjectParser(gson)); + transport.createRequestFactory(), new XmlObjectParser(new XmlMapper())); httpStorageOptions = HttpStorageOptions.newBuilder().setProjectId("test-project").build(); } From 86a9ab98728ac9c83fda0f6a3ca8538cfafd1f8c Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 17:02:36 +0000 Subject: [PATCH 09/18] removing mockito conifg no longer required --- .../resources/mockito-extensions/org.mockito.plugins.MockMaker | 1 - 1 file changed, 1 deletion(-) delete mode 100644 google-cloud-storage/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/google-cloud-storage/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/google-cloud-storage/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8..0000000000 --- a/google-cloud-storage/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file From 1863d0219b65fbb32ff012214bb44655b1971b62 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 17:21:51 +0000 Subject: [PATCH 10/18] chore: added missing library --- google-cloud-storage/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 855a1f999a..d043318d07 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -23,6 +23,10 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml + + com.fasterxml.jackson.core + jackson-databind + com.google.guava guava From 81648dcaf4942ba2a8d41e6a7c040e4dfa87ef02 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 18:46:34 +0000 Subject: [PATCH 11/18] fixing licence number --- .../cloud/storage/XmlObjectParserTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java new file mode 100644 index 0000000000..38dab1e530 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +public class XmlObjectParserTest { + + @Mock private XmlMapper xmlMapper; + + private XmlObjectParser xmlObjectParser; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + xmlObjectParser = new XmlObjectParser(xmlMapper); + } + + @Test + public void testParseAndClose() throws IOException { + InputStream in = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); + TestXmlObject expected = new TestXmlObject(); + when(xmlMapper.readValue(in, TestXmlObject.class)).thenReturn(expected); + TestXmlObject actual = xmlObjectParser.parseAndClose(in, StandardCharsets.UTF_8, TestXmlObject.class); + assertThat(actual).isSameInstanceAs(expected); + } + + private static class TestXmlObject {} +} From 2a5d1a02b8261c265c2a743a8b9567c108c0e05c Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Wed, 22 Oct 2025 18:47:40 +0000 Subject: [PATCH 12/18] fixing lint error --- .../java/com/google/cloud/storage/XmlObjectParserTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java index 38dab1e530..2780309e81 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java @@ -46,7 +46,8 @@ public void testParseAndClose() throws IOException { InputStream in = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); TestXmlObject expected = new TestXmlObject(); when(xmlMapper.readValue(in, TestXmlObject.class)).thenReturn(expected); - TestXmlObject actual = xmlObjectParser.parseAndClose(in, StandardCharsets.UTF_8, TestXmlObject.class); + TestXmlObject actual = + xmlObjectParser.parseAndClose(in, StandardCharsets.UTF_8, TestXmlObject.class); assertThat(actual).isSameInstanceAs(expected); } From 744db03c0dbab6acdee98f2440dfa3855e8188bc Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Thu, 23 Oct 2025 05:36:03 +0000 Subject: [PATCH 13/18] chore: adding fixes for review comments --- .../google/cloud/storage/XmlObjectParser.java | 16 ++++++++--- .../model/CreateMultipartUploadRequest.java | 1 - .../model}/ObjectLockMode.java | 2 +- ...MultipartUploadHttpRequestManagerTest.java | 27 +++++++++++-------- .../cloud/storage/XmlObjectParserTest.java | 13 +++++++-- 5 files changed, 40 insertions(+), 19 deletions(-) rename google-cloud-storage/src/main/java/com/google/cloud/storage/{ => multipartupload/model}/ObjectLockMode.java (97%) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java index 6893f53944..aae93d4796 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.Reader; import java.lang.reflect.Type; import java.nio.charset.Charset; @@ -35,21 +36,28 @@ public XmlObjectParser(XmlMapper xmlMapper) { @Override public T parseAndClose(InputStream in, Charset charset, Class dataClass) throws IOException { - return xmlMapper.readValue(in, dataClass); + return parseAndClose(new InputStreamReader(in, charset), dataClass); } @Override public Object parseAndClose(InputStream in, Charset charset, Type dataType) throws IOException { - return xmlMapper.readValue(in, xmlMapper.getTypeFactory().constructType(dataType)); + throw new UnsupportedOperationException( + "XmlObjectParse#" + + CrossTransportUtils.fmtMethodName( + "parseAndClose", InputStream.class, Charset.class, Type.class)); } @Override public T parseAndClose(Reader reader, Class dataClass) throws IOException { - return xmlMapper.readValue(reader, dataClass); + try (Reader r = reader) { + return xmlMapper.readValue(r, dataClass); + } } @Override public Object parseAndClose(Reader reader, Type dataType) throws IOException { - return xmlMapper.readValue(reader, xmlMapper.getTypeFactory().constructType(dataType)); + throw new UnsupportedOperationException( + "XmlObjectParse#" + + CrossTransportUtils.fmtMethodName("parseAndClose", Reader.class, Type.class)); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 722b140c24..97418e93fa 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -17,7 +17,6 @@ package com.google.cloud.storage.multipartupload.model; import com.google.api.core.BetaApi; -import com.google.cloud.storage.ObjectLockMode; import com.google.cloud.storage.Storage.PredefinedAcl; import com.google.cloud.storage.StorageClass; import com.google.common.base.MoreObjects; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ObjectLockMode.java similarity index 97% rename from google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java rename to google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ObjectLockMode.java index f08ec2e08f..a058719e1c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ObjectLockMode.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/ObjectLockMode.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.storage; +package com.google.cloud.storage.multipartupload.model; import com.google.api.core.ApiFunction; import com.google.api.core.BetaApi; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java index ff8ba131a0..0189056158 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.gson.GsonFactory; +import com.google.cloud.NoCredentials; import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler; import com.google.cloud.storage.it.runner.StorageITRunner; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -31,6 +31,7 @@ import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.cloud.storage.multipartupload.model.ObjectLockMode; import com.google.common.collect.ImmutableMap; import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; import io.grpc.netty.shaded.io.netty.buffer.Unpooled; @@ -50,7 +51,7 @@ @SingleBackend(Backend.PROD) @ParallelFriendly public final class ITMultipartUploadHttpRequestManagerTest { - private static final GsonFactory gson = GsonFactory.getDefaultInstance(); + private static final XmlMapper xmlMapper = new XmlMapper(); private static final NetHttpTransport transport = new NetHttpTransport.Builder().build(); private MultipartUploadHttpRequestManager multipartUploadHttpRequestManager; private HttpStorageOptions httpStorageOptions; @@ -62,7 +63,11 @@ public void setUp() throws Exception { multipartUploadHttpRequestManager = new MultipartUploadHttpRequestManager( transport.createRequestFactory(), new XmlObjectParser(new XmlMapper())); - httpStorageOptions = HttpStorageOptions.newBuilder().setProjectId("test-project").build(); + httpStorageOptions = + HttpStorageOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .build(); } @Test @@ -75,7 +80,7 @@ public void sendCreateMultipartUploadRequest_success() throws Exception { .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -143,7 +148,7 @@ public void sendCreateMultipartUploadRequest_withCannedAcl() throws Exception { .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -178,7 +183,7 @@ public void sendCreateMultipartUploadRequest_withMetadata() throws Exception { .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -212,7 +217,7 @@ public void sendCreateMultipartUploadRequest_withStorageClass() throws Exception .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -247,7 +252,7 @@ public void sendCreateMultipartUploadRequest_withKmsKeyName() throws Exception { .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -281,7 +286,7 @@ public void sendCreateMultipartUploadRequest_withObjectLockMode() throws Excepti .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -316,7 +321,7 @@ public void sendCreateMultipartUploadRequest_withObjectLockRetainUntilDate() thr .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); @@ -350,7 +355,7 @@ public void sendCreateMultipartUploadRequest_withCustomTime() throws Exception { .key("test-key") .uploadId("test-upload-id") .build(); - ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(response)); + ByteBuf buf = Unpooled.wrappedBuffer(xmlMapper.writeValueAsBytes(response)); DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK, buf); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java index 2780309e81..c4acd8c64a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/XmlObjectParserTest.java @@ -17,13 +17,16 @@ package com.google.cloud.storage; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.Reader; import java.nio.charset.StandardCharsets; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; @@ -33,19 +36,25 @@ public class XmlObjectParserTest { @Mock private XmlMapper xmlMapper; + private AutoCloseable mocks; private XmlObjectParser xmlObjectParser; @Before public void setUp() { - MockitoAnnotations.initMocks(this); + mocks = MockitoAnnotations.openMocks(this); xmlObjectParser = new XmlObjectParser(xmlMapper); } + @After + public void tearDown() throws Exception { + mocks.close(); + } + @Test public void testParseAndClose() throws IOException { InputStream in = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); TestXmlObject expected = new TestXmlObject(); - when(xmlMapper.readValue(in, TestXmlObject.class)).thenReturn(expected); + when(xmlMapper.readValue(any(Reader.class), any(Class.class))).thenReturn(expected); TestXmlObject actual = xmlObjectParser.parseAndClose(in, StandardCharsets.UTF_8, TestXmlObject.class); assertThat(actual).isSameInstanceAs(expected); From e565af605ca118749fbda89a4843f05cbcdb93f6 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Thu, 23 Oct 2025 08:21:13 +0000 Subject: [PATCH 14/18] chore: Adding fixes for IT. --- .../ITMultipartUploadHttpRequestManagerTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java index 0189056158..0a5b92fc9b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertThrows; import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.cloud.NoCredentials; import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler; @@ -127,13 +128,11 @@ public void sendCreateMultipartUploadRequest_error() throws Exception { .contentType("application/octet-stream") .build(); - StorageException se = - assertThrows( - StorageException.class, - () -> - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions)); - assertThat(se.getCode()).isEqualTo(400); + assertThrows( + HttpResponseException.class, + () -> + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( + endpoint, request, httpStorageOptions)); } } @@ -141,7 +140,7 @@ public void sendCreateMultipartUploadRequest_error() throws Exception { public void sendCreateMultipartUploadRequest_withCannedAcl() throws Exception { HttpRequestHandler handler = req -> { - assertThat(req.headers().get("x-goog-acl")).isEqualTo("authenticatedRead"); + assertThat(req.headers().get("x-goog-acl")).isEqualTo("AUTHENTICATED_READ"); CreateMultipartUploadResponse response = CreateMultipartUploadResponse.builder() .bucket("test-bucket") From b8530f75af08e4cf28f3e12d6bc0994e9ad10645 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 24 Oct 2025 12:58:28 +0000 Subject: [PATCH 15/18] chore: Adding review comments --- .../cloud/storage/MultipartUploadClient.java | 3 +- .../storage/MultipartUploadClientImpl.java | 13 +- .../MultipartUploadHttpRequestManager.java | 115 ++++++++++++------ .../java/com/google/cloud/storage/Utils.java | 5 + ...MultipartUploadHttpRequestManagerTest.java | 55 ++++----- 5 files changed, 112 insertions(+), 79 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java index 0d1a560034..768eed5455 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java @@ -46,8 +46,7 @@ public static MultipartUploadClient create(MultipartUploadSettings config) { HttpStorageOptions options = config.getOptions(); return new MultipartUploadClientImpl( URI.create(options.getHost()), - options.getStorageRpcV1().getStorage().getRequestFactory(), options.createRetrier(), - options); + MultipartUploadHttpRequestManager.createFrom(options)); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index 8c333455a7..ab3a8a0de9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -15,8 +15,6 @@ */ package com.google.cloud.storage; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.google.api.client.http.HttpRequestFactory; import com.google.api.core.BetaApi; import com.google.cloud.storage.Retrying.Retrier; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; @@ -32,15 +30,14 @@ final class MultipartUploadClientImpl extends MultipartUploadClient { private final MultipartUploadHttpRequestManager httpRequestManager; - private final HttpStorageOptions options; private final Retrier retrier; private final URI uri; MultipartUploadClientImpl( - URI uri, HttpRequestFactory requestFactory, Retrier retrier, HttpStorageOptions options) { - this.httpRequestManager = - new MultipartUploadHttpRequestManager(requestFactory, new XmlObjectParser(new XmlMapper())); - this.options = options; + URI uri, + Retrier retrier, + MultipartUploadHttpRequestManager multipartUploadHttpRequestManager) { + this.httpRequestManager = multipartUploadHttpRequestManager; this.retrier = retrier; this.uri = uri; } @@ -48,6 +45,6 @@ final class MultipartUploadClientImpl extends MultipartUploadClient { @BetaApi public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) throws IOException { - return httpRequestManager.sendCreateMultipartUploadRequest(uri, request, options); + return httpRequestManager.sendCreateMultipartUploadRequest(uri, request); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java index 78e5b19520..fc7b181a86 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java @@ -15,105 +15,144 @@ */ package com.google.cloud.storage; +import static com.google.cloud.storage.Utils.ifNonNull; + +import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.util.ObjectParser; +import com.google.api.gax.core.GaxProperties; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.api.gax.rpc.HeaderProvider; +import com.google.api.services.storage.Storage; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest; import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse; +import com.google.common.base.StandardSystemProperty; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; final class MultipartUploadHttpRequestManager { private final HttpRequestFactory requestFactory; private final ObjectParser objectParser; + private final HeaderProvider headerProvider; - MultipartUploadHttpRequestManager(HttpRequestFactory requestFactory, ObjectParser objectParser) { + MultipartUploadHttpRequestManager( + HttpRequestFactory requestFactory, ObjectParser objectParser, HeaderProvider headerProvider) { this.requestFactory = requestFactory; this.objectParser = objectParser; + this.headerProvider = headerProvider; } CreateMultipartUploadResponse sendCreateMultipartUploadRequest( - URI uri, CreateMultipartUploadRequest request, HttpStorageOptions options) - throws IOException { + URI uri, CreateMultipartUploadRequest request) throws IOException { - String encodedBucket = encode(request.bucket()); - String encodedKey = encode(request.key()); + String encodedBucket = urlEncode(request.bucket()); + String encodedKey = urlEncode(request.key()); String resourcePath = "/" + encodedBucket + "/" + encodedKey; String createUri = uri.toString() + resourcePath + "?uploads"; HttpRequest httpRequest = requestFactory.buildPostRequest( new GenericUrl(createUri), new ByteArrayContent(request.getContentType(), new byte[0])); - httpRequest.getHeaders().setContentType(request.getContentType()); - for (Map.Entry entry : - getExtensionHeadersForCreateMultipartUpload(request, options).entrySet()) { - httpRequest.getHeaders().set(entry.getKey(), entry.getValue()); - } + httpRequest.getHeaders().putAll(headerProvider.getHeaders()); + addHeadersForCreateMultipartUpload(request, httpRequest.getHeaders()); httpRequest.setParser(objectParser); httpRequest.setThrowExceptionOnExecuteError(true); return httpRequest.execute().parseAs(CreateMultipartUploadResponse.class); } - private Map getExtensionHeadersForCreateMultipartUpload( - CreateMultipartUploadRequest request, HttpStorageOptions options) { - Map extensionHeaders = getGenericExtensionHeader(options); + @SuppressWarnings("DataFlowIssue") + static MultipartUploadHttpRequestManager createFrom(HttpStorageOptions options) { + Storage storage = options.getStorageRpcV1().getStorage(); + ImmutableMap.Builder stableHeaders = + ImmutableMap.builder() + // http-java-client will automatically append its own version to the user-agent + .put("User-Agent", "gcloud-java/" + options.getLibraryVersion()) + .put( + "x-goog-api-client", + String.format( + "gl-java/%s gccl/%s %s/%s", + GaxProperties.getJavaVersion(), + options.getLibraryVersion(), + formatName(StandardSystemProperty.OS_NAME.value()), + formatSemver(StandardSystemProperty.OS_VERSION.value()))); + ifNonNull(options.getProjectId(), pid -> stableHeaders.put("x-goog-user-project", pid)); + return new MultipartUploadHttpRequestManager( + storage.getRequestFactory(), + new XmlObjectParser(new XmlMapper()), + options.getMergedHeaderProvider(FixedHeaderProvider.create(stableHeaders.build()))); + } + + private void addHeadersForCreateMultipartUpload( + CreateMultipartUploadRequest request, HttpHeaders headers) { + // TODO(shreyassinha): add a PredefinedAcl::getXmlEntry with the corresponding value from + // https://cloud.google.com/storage/docs/xml-api/reference-headers#xgoogacl if (request.getCannedAcl() != null) { - extensionHeaders.put("x-goog-acl", request.getCannedAcl().toString()); + headers.put("x-goog-acl", request.getCannedAcl().toString()); } // TODO(shreyassinha) Add encoding for x-goog-meta-* headers if (request.getMetadata() != null) { for (Map.Entry entry : request.getMetadata().entrySet()) { if (entry.getKey() != null || entry.getValue() != null) { - extensionHeaders.put("x-goog-meta-" + entry.getKey(), entry.getValue()); + headers.put("x-goog-meta-" + entry.getKey(), entry.getValue()); } } } if (request.getStorageClass() != null) { - extensionHeaders.put("x-goog-storage-class", request.getStorageClass().toString()); + headers.put("x-goog-storage-class", request.getStorageClass().toString()); } if (request.getKmsKeyName() != null && !request.getKmsKeyName().isEmpty()) { - extensionHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); + headers.put("x-goog-encryption-kms-key-name", request.getKmsKeyName()); } if (request.getObjectLockMode() != null) { - extensionHeaders.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); + headers.put("x-goog-object-lock-mode", request.getObjectLockMode().toString()); } if (request.getObjectLockRetainUntilDate() != null) { - extensionHeaders.put( + headers.put( "x-goog-object-lock-retain-until-date", - toRfc3339String(request.getObjectLockRetainUntilDate())); + Utils.offsetDateTimeRfc3339Codec.encode(request.getObjectLockRetainUntilDate())); } if (request.getCustomTime() != null) { - extensionHeaders.put("x-goog-custom-time", toRfc3339String(request.getCustomTime())); + headers.put( + "x-goog-custom-time", Utils.offsetDateTimeRfc3339Codec.encode(request.getCustomTime())); } - return extensionHeaders; } - private Map getGenericExtensionHeader(HttpStorageOptions options) { - Map extensionHeaders = new HashMap<>(); - if (options.getClientLibToken() != null) { - extensionHeaders.put("x-goog-api-client", options.getClientLibToken()); - } - if (options.getProjectId() != null) { - extensionHeaders.put("x-goog-user-project", options.getProjectId()); - } - return extensionHeaders; + private static String urlEncode(String value) throws UnsupportedEncodingException { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); } - private String encode(String value) throws UnsupportedEncodingException { - return URLEncoder.encode(value, StandardCharsets.UTF_8.name()); + private static String formatName(String name) { + // Only lowercase letters, digits, and "-" are allowed + return name.toLowerCase().replaceAll("[^\\w\\d\\-]", "-"); + } + + private static String formatSemver(String version) { + return formatSemver(version, version); } - private String toRfc3339String(OffsetDateTime dateTime) { - return DateTimeFormatter.ISO_INSTANT.format(dateTime); + private static String formatSemver(String version, String defaultValue) { + if (version == null) { + return null; + } + + // Take only the semver version: x.y.z-a_b_c -> x.y.z + Matcher m = Pattern.compile("(\\d+\\.\\d+\\.\\d+).*").matcher(version); + if (m.find()) { + return m.group(1); + } else { + return defaultValue; + } } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java index eece2fe79d..90d6122c9a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java @@ -84,6 +84,11 @@ final class Utils { static final Codec durationSecondsCodec = Codec.of(Duration::getSeconds, Duration::ofSeconds); + static final Codec offsetDateTimeRfc3339Codec = + Codec.of( + RFC_3339_DATE_TIME_FORMATTER::format, + s -> OffsetDateTime.parse(s, RFC_3339_DATE_TIME_FORMATTER)); + @VisibleForTesting static final Codec dateTimeCodec = Codec.of( diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java index 0a5b92fc9b..72625d0baf 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITMultipartUploadHttpRequestManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.google.api.client.http.HttpResponseException; -import com.google.api.client.http.javanet.NetHttpTransport; import com.google.cloud.NoCredentials; import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler; import com.google.cloud.storage.it.runner.StorageITRunner; @@ -53,22 +52,18 @@ @ParallelFriendly public final class ITMultipartUploadHttpRequestManagerTest { private static final XmlMapper xmlMapper = new XmlMapper(); - private static final NetHttpTransport transport = new NetHttpTransport.Builder().build(); private MultipartUploadHttpRequestManager multipartUploadHttpRequestManager; - private HttpStorageOptions httpStorageOptions; - @Rule public final TemporaryFolder temp = new TemporaryFolder(); @Before public void setUp() throws Exception { - multipartUploadHttpRequestManager = - new MultipartUploadHttpRequestManager( - transport.createRequestFactory(), new XmlObjectParser(new XmlMapper())); - httpStorageOptions = + HttpStorageOptions httpStorageOptions = HttpStorageOptions.newBuilder() .setProjectId("test-project") .setCredentials(NoCredentials.getInstance()) .build(); + multipartUploadHttpRequestManager = + MultipartUploadHttpRequestManager.createFrom(httpStorageOptions); } @Test @@ -99,8 +94,7 @@ public void sendCreateMultipartUploadRequest_success() throws Exception { .build(); CreateMultipartUploadResponse response = - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); assertThat(response).isNotNull(); assertThat(response.bucket()).isEqualTo("test-bucket"); @@ -132,7 +126,7 @@ public void sendCreateMultipartUploadRequest_error() throws Exception { HttpResponseException.class, () -> multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions)); + endpoint, request)); } } @@ -165,8 +159,7 @@ public void sendCreateMultipartUploadRequest_withCannedAcl() throws Exception { .cannedAcl(Storage.PredefinedAcl.AUTHENTICATED_READ) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @@ -200,8 +193,7 @@ public void sendCreateMultipartUploadRequest_withMetadata() throws Exception { .metadata(ImmutableMap.of("key1", "value1", "key2", "value2")) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @@ -234,8 +226,7 @@ public void sendCreateMultipartUploadRequest_withStorageClass() throws Exception .storageClass(StorageClass.ARCHIVE) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @@ -269,8 +260,7 @@ public void sendCreateMultipartUploadRequest_withKmsKeyName() throws Exception { .kmsKeyName("projects/p/locations/l/keyRings/r/cryptoKeys/k") .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @@ -303,17 +293,19 @@ public void sendCreateMultipartUploadRequest_withObjectLockMode() throws Excepti .objectLockMode(ObjectLockMode.GOVERNANCE) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @Test public void sendCreateMultipartUploadRequest_withObjectLockRetainUntilDate() throws Exception { + OffsetDateTime retainUtil = OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); HttpRequestHandler handler = req -> { - assertThat(req.headers().get("x-goog-object-lock-retain-until-date")) - .isEqualTo("2024-01-01T00:00:00Z"); + OffsetDateTime actual = + Utils.offsetDateTimeRfc3339Codec.decode( + req.headers().get("x-goog-object-lock-retain-until-date")); + assertThat(actual).isEqualTo(retainUtil); CreateMultipartUploadResponse response = CreateMultipartUploadResponse.builder() .bucket("test-bucket") @@ -335,19 +327,21 @@ public void sendCreateMultipartUploadRequest_withObjectLockRetainUntilDate() thr .bucket("test-bucket") .key("test-key") .contentType("application/octet-stream") - .objectLockRetainUntilDate(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)) + .objectLockRetainUntilDate(retainUtil) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } @Test public void sendCreateMultipartUploadRequest_withCustomTime() throws Exception { + OffsetDateTime customTime = OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); HttpRequestHandler handler = req -> { - assertThat(req.headers().get("x-goog-custom-time")).isEqualTo("2024-01-01T00:00:00Z"); + OffsetDateTime actual = + Utils.offsetDateTimeRfc3339Codec.decode(req.headers().get("x-goog-custom-time")); + assertThat(actual).isEqualTo(customTime); CreateMultipartUploadResponse response = CreateMultipartUploadResponse.builder() .bucket("test-bucket") @@ -369,11 +363,10 @@ public void sendCreateMultipartUploadRequest_withCustomTime() throws Exception { .bucket("test-bucket") .key("test-key") .contentType("application/octet-stream") - .customTime(OffsetDateTime.of(2024, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)) + .customTime(customTime) .build(); - multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest( - endpoint, request, httpStorageOptions); + multipartUploadHttpRequestManager.sendCreateMultipartUploadRequest(endpoint, request); } } } From 956c59505b028924d78e75d952cdecb1461e7d3c Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 24 Oct 2025 13:03:50 +0000 Subject: [PATCH 16/18] Adding review comments --- google-cloud-storage/pom.xml | 4 ---- .../com/google/cloud/storage/MultipartUploadClientImpl.java | 2 +- .../cloud/storage/MultipartUploadHttpRequestManager.java | 2 +- .../main/java/com/google/cloud/storage/XmlObjectParser.java | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index d043318d07..855a1f999a 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -23,10 +23,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - - com.fasterxml.jackson.core - jackson-databind - com.google.guava guava diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java index ab3a8a0de9..853186e775 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java index fc7b181a86..eb8db6cc60 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadHttpRequestManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java index aae93d4796..27c91fef48 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/XmlObjectParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Google LLC + * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From b8437d6d7c35568c61b135f46047af1160e4067a Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 24 Oct 2025 13:34:39 +0000 Subject: [PATCH 17/18] chore: Adding missing BetaApi annotation. --- .../multipartupload/model/CreateMultipartUploadRequest.java | 2 ++ .../multipartupload/model/CreateMultipartUploadResponse.java | 1 + 2 files changed, 3 insertions(+) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index 97418e93fa..dc1545d1f4 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -206,6 +206,8 @@ public static Builder builder() { return new Builder(); } + + @BetaApi public static final class Builder { private String bucket; private String key; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java index 35d2b2c171..5487dd7316 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadResponse.java @@ -113,6 +113,7 @@ public static Builder builder() { } /** A builder for {@link CreateMultipartUploadResponse} objects. */ + @BetaApi public static final class Builder { private String bucket; private String key; From 1ec5eadef483f7bcdd3b64dc42f3825fc1e36706 Mon Sep 17 00:00:00 2001 From: Shreyas Sinha Date: Fri, 24 Oct 2025 13:46:42 +0000 Subject: [PATCH 18/18] chore:fixing lint error. --- .../multipartupload/model/CreateMultipartUploadRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java index dc1545d1f4..44161ad160 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/multipartupload/model/CreateMultipartUploadRequest.java @@ -206,7 +206,6 @@ public static Builder builder() { return new Builder(); } - @BetaApi public static final class Builder { private String bucket;