diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java index 0a1480adff02f..db94bbbb75058 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/HeaderPreprocessor.java @@ -17,39 +17,60 @@ */ package org.apache.hadoop.ozone.s3; +import javax.annotation.Priority; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerRequestFilter; import javax.ws.rs.container.PreMatching; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.ext.Provider; import java.io.IOException; /** * Filter to adjust request headers for compatible reasons. + * + * It should be executed AFTER signature check (VirtualHostStyleFilter) as the + * original Content-Type could be part of the base of the signature. */ - @Provider @PreMatching +@Priority(VirtualHostStyleFilter.PRIORITY + + S3GatewayHttpServer.FILTER_PRIORITY_DO_AFTER) public class HeaderPreprocessor implements ContainerRequestFilter { + public static final String MULTIPART_UPLOAD_MARKER = "ozone/mpu"; + @Override public void filter(ContainerRequestContext requestContext) throws IOException { - if (requestContext.getUriInfo().getQueryParameters() - .containsKey("delete")) { + MultivaluedMap queryParameters = + requestContext.getUriInfo().getQueryParameters(); + + if (queryParameters.containsKey("delete")) { //aws cli doesn't send proper Content-Type and by default POST requests //processed as form-url-encoded. Here we can fix this. requestContext.getHeaders() .putSingle("Content-Type", MediaType.APPLICATION_XML); } - if (requestContext.getUriInfo().getQueryParameters() - .containsKey("uploadId")) { + if (queryParameters.containsKey("uploadId")) { //aws cli doesn't send proper Content-Type and by default POST requests //processed as form-url-encoded. Here we can fix this. requestContext.getHeaders() .putSingle("Content-Type", MediaType.APPLICATION_XML); + } else if (queryParameters.containsKey("uploads")) { + // uploads defined but uploadId is not --> this is the creation of the + // multi-part-upload requests. + // + //In AWS SDK for go uses application/octet-stream which also + //should be fixed to route the request to the right jaxrs method. + // + //Should be empty instead of XML as the body is empty which can not be + //serialized as as CompleteMultipartUploadRequest + requestContext.getHeaders() + .putSingle("Content-Type", MULTIPART_UPLOAD_MARKER); } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java index f20b928b1f952..f3d83412ae647 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java @@ -27,6 +27,11 @@ */ public class S3GatewayHttpServer extends BaseHttpServer { + /** + * Default offset between two filters. + */ + public static final int FILTER_PRIORITY_DO_AFTER = 50; + public S3GatewayHttpServer(Configuration conf, String name) throws IOException { super(conf, name); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java index 50014fea926f2..9ce98e11ee140 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.s3; +import javax.annotation.Priority; import javax.inject.Inject; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.container.ContainerRequestFilter; @@ -46,8 +47,11 @@ @Provider @PreMatching +@Priority(VirtualHostStyleFilter.PRIORITY) public class VirtualHostStyleFilter implements ContainerRequestFilter { + public static final int PRIORITY = 100; + private static final Logger LOG = LoggerFactory.getLogger( VirtualHostStyleFilter.class); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index b520e7be0bf3a..70bfb7f8e063f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.s3.endpoint; +import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -58,6 +59,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.SignedChunksInputStream; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -417,33 +419,19 @@ public Response delete( } + /** + * Initialize MultiPartUpload request. + *

+ * Note: the specific content type is set by the HeaderPreprocessor. + */ @POST @Produces(MediaType.APPLICATION_XML) - public Response multipartUpload( + @Consumes(HeaderPreprocessor.MULTIPART_UPLOAD_MARKER) + public Response initializeMultipartUpload( @PathParam("bucket") String bucket, - @PathParam("path") String key, - @QueryParam("uploads") String uploads, - @QueryParam("uploadId") @DefaultValue("") String uploadID, - CompleteMultipartUploadRequest request) throws IOException, OS3Exception { - if (!uploadID.equals("")) { - //Complete Multipart upload request. - return completeMultipartUpload(bucket, key, uploadID, request); - } else { - // Initiate Multipart upload request. - return initiateMultipartUpload(bucket, key); - } - } - - /** - * Initiate Multipart upload request. - * @param bucket - * @param key - * @return Response - * @throws IOException - * @throws OS3Exception - */ - private Response initiateMultipartUpload(String bucket, String key) throws - IOException, OS3Exception { + @PathParam("path") String key + ) + throws IOException, OS3Exception { try { OzoneBucket ozoneBucket = getBucket(bucket); String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER); @@ -473,7 +461,6 @@ private Response initiateMultipartUpload(String bucket, String key) throws multipartUploadInitiateResponse.setKey(key); multipartUploadInitiateResponse.setUploadID(multipartInfo.getUploadID()); - return Response.status(Status.OK).entity( multipartUploadInitiateResponse).build(); } catch (IOException ex) { @@ -484,18 +471,15 @@ private Response initiateMultipartUpload(String bucket, String key) throws } /** - * Complete Multipart upload request. - * @param bucket - * @param key - * @param uploadID - * @param multipartUploadRequest - * @return Response - * @throws IOException - * @throws OS3Exception + * Complete a multipart upload. */ - private Response completeMultipartUpload(String bucket, String key, String - uploadID, CompleteMultipartUploadRequest multipartUploadRequest) throws - IOException, OS3Exception { + @POST + @Produces(MediaType.APPLICATION_XML) + public Response completeMultipartUpload(@PathParam("bucket") String bucket, + @PathParam("path") String key, + @QueryParam("uploadId") @DefaultValue("") String uploadID, + CompleteMultipartUploadRequest multipartUploadRequest) + throws IOException, OS3Exception { OzoneBucket ozoneBucket = getBucket(bucket); Map partsMap = new TreeMap<>(); List partList = diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java index 76d4a12b7bbd1..912a769cd3f82 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java @@ -56,7 +56,7 @@ public void testAbortMultipartUpload() throws Exception { rest.setHeaders(headers); rest.setClient(client); - Response response = rest.multipartUpload(bucket, key, "", "", null); + Response response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); MultipartUploadInitiateResponse multipartUploadInitiateResponse = diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java index 6f48ecb509a8b..212721af00f91 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java @@ -60,7 +60,7 @@ public void testInitiateMultipartUpload() throws Exception { rest.setHeaders(headers); rest.setClient(client); - Response response = rest.multipartUpload(bucket, key, "", "", null); + Response response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); MultipartUploadInitiateResponse multipartUploadInitiateResponse = @@ -69,7 +69,7 @@ public void testInitiateMultipartUpload() throws Exception { String uploadID = multipartUploadInitiateResponse.getUploadID(); // Calling again should return different uploadID. - response = rest.multipartUpload(bucket, key, "", "", null); + response = rest.initializeMultipartUpload(bucket, key); assertEquals(response.getStatus(), 200); multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java index ac6aa72e4f188..21545ec9b0704 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java @@ -61,7 +61,7 @@ public static void setUp() throws Exception { REST.setHeaders(headers); REST.setClient(client); - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 8be61310f99b2..b9e3885ac6f3c 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -70,7 +70,7 @@ public static void setUp() throws Exception { private String initiateMultipartUpload(String key) throws IOException, OS3Exception { - Response response = REST.multipartUpload(BUCKET, key, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, key); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); @@ -99,7 +99,7 @@ private Part uploadPart(String key, String uploadID, int partNumber, String private void completeMultipartUpload(String key, CompleteMultipartUploadRequest completeMultipartUploadRequest, String uploadID) throws IOException, OS3Exception { - Response response = REST.multipartUpload(BUCKET, key, "", uploadID, + Response response = REST.completeMultipartUpload(BUCKET, key, uploadID, completeMultipartUploadRequest); assertEquals(response.getStatus(), 200); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java index 120fbb2f2ee56..3e91a77ffd4ae 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPartUpload.java @@ -67,7 +67,7 @@ public static void setUp() throws Exception { @Test public void testPartUpload() throws Exception { - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID()); @@ -86,7 +86,7 @@ public void testPartUpload() throws Exception { @Test public void testPartUploadWithOverride() throws Exception { - Response response = REST.multipartUpload(BUCKET, KEY, "", "", null); + Response response = REST.initializeMultipartUpload(BUCKET, KEY); MultipartUploadInitiateResponse multipartUploadInitiateResponse = (MultipartUploadInitiateResponse) response.getEntity(); assertNotNull(multipartUploadInitiateResponse.getUploadID());