From c638d8ee980236dc7abb114b085c0b6927e9726f Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Thu, 22 Aug 2019 17:38:39 -0300 Subject: [PATCH 1/4] HBASE-22846 Internal Error 500 when Using HBASE REST API to Create Namespace. --- .../rest/NamespacesInstanceResource.java | 69 +++++-------------- .../rest/model/NamespacesInstanceModel.java | 14 ++++ 2 files changed, 30 insertions(+), 53 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java index 3ff25f99ef77..215abaea0b33 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java @@ -135,35 +135,9 @@ public Response get(final @Context ServletContext context, @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF}) public Response put(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { - if (LOG.isTraceEnabled()) { - LOG.trace("PUT " + uriInfo.getAbsolutePath()); - } - servlet.getMetrics().incrementRequests(1); return processUpdate(model, true, uriInfo); } - /** - * Build a response for PUT alter namespace with no properties specified. - * @param message value not used. - * @param headers value not used. - * @return response code. - */ - @PUT - public Response putNoBody(final byte[] message, - final @Context UriInfo uriInfo, final @Context HttpHeaders headers) { - if (LOG.isTraceEnabled()) { - LOG.trace("PUT " + uriInfo.getAbsolutePath()); - } - servlet.getMetrics().incrementRequests(1); - try{ - NamespacesInstanceModel model = new NamespacesInstanceModel(namespace); - return processUpdate(model, true, uriInfo); - }catch(IOException ioe){ - servlet.getMetrics().incrementFailedPutRequests(1); - throw new RuntimeException("Cannot retrieve info for '" + namespace + "'."); - } - } - /** * Build a response for POST create namespace with properties specified. * @param model properties used for create. @@ -175,39 +149,26 @@ public Response putNoBody(final byte[] message, MIMETYPE_PROTOBUF_IETF}) public Response post(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { - - if (LOG.isTraceEnabled()) { - LOG.trace("POST " + uriInfo.getAbsolutePath()); - } - servlet.getMetrics().incrementRequests(1); return processUpdate(model, false, uriInfo); } - /** - * Build a response for POST create namespace with no properties specified. - * @param message value not used. - * @param headers value not used. - * @return response code. - */ - @POST - public Response postNoBody(final byte[] message, - final @Context UriInfo uriInfo, final @Context HttpHeaders headers) { + + // Check that POST or PUT is valid and then update namespace. + private Response processUpdate(NamespacesInstanceModel model, final boolean updateExisting, + final UriInfo uriInfo) { if (LOG.isTraceEnabled()) { - LOG.trace("POST " + uriInfo.getAbsolutePath()); + LOG.trace((updateExisting ? "PUT " : "POST ") + uriInfo.getAbsolutePath()); } - servlet.getMetrics().incrementRequests(1); - try{ - NamespacesInstanceModel model = new NamespacesInstanceModel(namespace); - return processUpdate(model, false, uriInfo); - }catch(IOException ioe){ - servlet.getMetrics().incrementFailedPutRequests(1); - throw new RuntimeException("Cannot retrieve info for '" + namespace + "'."); + if (model == null) { + try { + model = new NamespacesInstanceModel(namespace); + } catch(IOException ioe) { + servlet.getMetrics().incrementFailedPutRequests(1); + throw new RuntimeException("Cannot retrieve info for '" + namespace + "'."); + } } - } + servlet.getMetrics().incrementRequests(1); - // Check that POST or PUT is valid and then update namespace. - private Response processUpdate(final NamespacesInstanceModel model, final boolean updateExisting, - final UriInfo uriInfo) { if (servlet.isReadOnly()) { servlet.getMetrics().incrementFailedPutRequests(1); return Response.status(Response.Status.FORBIDDEN).type(MIMETYPE_TEXT) @@ -265,7 +226,9 @@ private Response createOrUpdate(final NamespacesInstanceModel model, final UriIn } servlet.getMetrics().incrementSucessfulPutRequests(1); - return Response.created(uriInfo.getAbsolutePath()).build(); + + return updateExisting ? Response.ok(uriInfo.getAbsolutePath()).build() : + Response.created(uriInfo.getAbsolutePath()).build(); } private boolean doesNamespaceExist(Admin admin, String namespaceName) throws IOException{ diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java index 022ec38d3bfe..ccf959cf18f8 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java @@ -30,6 +30,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import com.google.gson.Gson; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; @@ -166,4 +167,17 @@ public ProtobufMessageHandler getObjectFromMessage(byte[] message) throws IOExce } return this; } + + public static void main(String[] args) { + NamespacesInstanceModel model = new NamespacesInstanceModel(); + model.addProperty("test", "test value"); + model.addProperty("test1", "test value 1"); + model.namespaceName = "name"; + + Gson gson = new Gson(); + String json = gson.toJson(model); + System.out.println(json); + + + } } From 10ef12d6a07a78b4c2cef158cada4130b2b41b5a Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 30 Aug 2019 17:36:09 +0100 Subject: [PATCH 2/4] removed test main --- .../hbase/rest/model/NamespacesInstanceModel.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java index ccf959cf18f8..af3b0b067a43 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java @@ -30,7 +30,6 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; -import com.google.gson.Gson; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; @@ -168,16 +167,4 @@ public ProtobufMessageHandler getObjectFromMessage(byte[] message) throws IOExce return this; } - public static void main(String[] args) { - NamespacesInstanceModel model = new NamespacesInstanceModel(); - model.addProperty("test", "test value"); - model.addProperty("test1", "test value 1"); - model.namespaceName = "name"; - - Gson gson = new Gson(); - String json = gson.toJson(model); - System.out.println(json); - - - } } From d0c00cad4f406a8281e119c63848bff14d146544 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Mon, 2 Sep 2019 10:10:41 +0100 Subject: [PATCH 3/4] temp changes --- .../hadoop/hbase/rest/NamespacesInstanceResource.java | 4 ++-- .../hadoop/hbase/rest/TestNamespacesInstanceResource.java | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java index 215abaea0b33..6f7045c4e036 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java @@ -132,7 +132,7 @@ public Response get(final @Context ServletContext context, * @return response code. */ @PUT - @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + @Consumes({MIMETYPE_BINARY, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF}) public Response put(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { return processUpdate(model, true, uriInfo); @@ -146,7 +146,7 @@ public Response put(final NamespacesInstanceModel model, final @Context UriInfo */ @POST @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, - MIMETYPE_PROTOBUF_IETF}) + MIMETYPE_PROTOBUF_IETF, MIMETYPE_BINARY}) public Response post(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { return processUpdate(model, false, uriInfo); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java index e0beeab49092..cefb3ad37b59 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.http.Header; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -399,9 +400,14 @@ public void testNamespaceCreateAndDeletePBAndNoBody() throws IOException, JAXBEx assertNull(nd4); conf.set("hbase.rest.readonly", "false"); - // Create namespace via no body and protobuf. + // Create namespace with no body and binary content type. response = client.post(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); assertEquals(201, response.getCode()); + // Create namespace with no body and no content type. + Header[] nullHeaders = null; + response = client.post(namespacePath3, nullHeaders, new byte[]{}); + assertEquals(201, response.getCode()); + // Create namespace with protobuf. response = client.post(namespacePath4, Constants.MIMETYPE_PROTOBUF, model4.createProtobufOutput()); assertEquals(201, response.getCode()); From 384ea471887a0e79c8bf44770d37ec1d7da61305 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Mon, 2 Sep 2019 17:11:18 +0100 Subject: [PATCH 4/4] fixing UT to avoid usage of unsupported content-type (application/octet-stream) --- .../rest/NamespacesInstanceResource.java | 4 +-- .../rest/TestNamespacesInstanceResource.java | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java index 6f7045c4e036..215abaea0b33 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/NamespacesInstanceResource.java @@ -132,7 +132,7 @@ public Response get(final @Context ServletContext context, * @return response code. */ @PUT - @Consumes({MIMETYPE_BINARY, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, + @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF}) public Response put(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { return processUpdate(model, true, uriInfo); @@ -146,7 +146,7 @@ public Response put(final NamespacesInstanceModel model, final @Context UriInfo */ @POST @Consumes({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, - MIMETYPE_PROTOBUF_IETF, MIMETYPE_BINARY}) + MIMETYPE_PROTOBUF_IETF}) public Response post(final NamespacesInstanceModel model, final @Context UriInfo uriInfo) { return processUpdate(model, false, uriInfo); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java index cefb3ad37b59..53eeecb48908 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestNamespacesInstanceResource.java @@ -330,6 +330,12 @@ public void testNamespaceCreateAndDeleteXMLAndJSON() throws IOException, JAXBExc jsonString = jsonMapper.writeValueAsString(model2); response = client.post(namespacePath2, Constants.MIMETYPE_JSON, Bytes.toBytes(jsonString)); assertEquals(201, response.getCode()); + //check passing null content-type with a payload returns 415 + Header[] nullHeaders = null; + response = client.post(namespacePath1, nullHeaders, toXML(model1)); + assertEquals(415, response.getCode()); + response = client.post(namespacePath1, nullHeaders, Bytes.toBytes(jsonString)); + assertEquals(415, response.getCode()); // Check that created namespaces correctly. nd1 = findNamespace(admin, NAMESPACE1); @@ -380,8 +386,12 @@ public void testNamespaceCreateAndDeletePBAndNoBody() throws IOException, JAXBEx model4 = testNamespacesInstanceModel.buildTestModel(NAMESPACE4, NAMESPACE4_PROPS); testNamespacesInstanceModel.checkModel(model4, NAMESPACE4, NAMESPACE4_PROPS); + //Defines null headers for use in tests where no body content is provided, so that we set + // no content-type in the request + Header[] nullHeaders = null; + // Test cannot PUT (alter) non-existent namespace. - response = client.put(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); + response = client.put(namespacePath3, nullHeaders, new byte[]{}); assertEquals(403, response.getCode()); response = client.put(namespacePath4, Constants.MIMETYPE_PROTOBUF, model4.createProtobufOutput()); @@ -389,7 +399,7 @@ public void testNamespaceCreateAndDeletePBAndNoBody() throws IOException, JAXBEx // Test cannot create tables when in read only mode. conf.set("hbase.rest.readonly", "true"); - response = client.post(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); + response = client.post(namespacePath3, nullHeaders, new byte[]{}); assertEquals(403, response.getCode()); response = client.put(namespacePath4, Constants.MIMETYPE_PROTOBUF, model4.createProtobufOutput()); @@ -401,16 +411,15 @@ public void testNamespaceCreateAndDeletePBAndNoBody() throws IOException, JAXBEx conf.set("hbase.rest.readonly", "false"); // Create namespace with no body and binary content type. - response = client.post(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); - assertEquals(201, response.getCode()); - // Create namespace with no body and no content type. - Header[] nullHeaders = null; response = client.post(namespacePath3, nullHeaders, new byte[]{}); assertEquals(201, response.getCode()); - // Create namespace with protobuf. + // Create namespace with protobuf content-type. response = client.post(namespacePath4, Constants.MIMETYPE_PROTOBUF, model4.createProtobufOutput()); assertEquals(201, response.getCode()); + //check setting unsupported content-type returns 415 + response = client.post(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); + assertEquals(415, response.getCode()); // Check that created namespaces correctly. nd3 = findNamespace(admin, NAMESPACE3); @@ -421,7 +430,7 @@ public void testNamespaceCreateAndDeletePBAndNoBody() throws IOException, JAXBEx checkNamespaceProperties(nd4, NAMESPACE4_PROPS); // Check cannot post tables that already exist. - response = client.post(namespacePath3, Constants.MIMETYPE_BINARY, new byte[]{}); + response = client.post(namespacePath3, nullHeaders, new byte[]{}); assertEquals(403, response.getCode()); response = client.post(namespacePath4, Constants.MIMETYPE_PROTOBUF, model4.createProtobufOutput());