From 7d4897f31574f1afb659c13b9066760ec98b2e50 Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Mon, 20 May 2019 15:38:03 -0700 Subject: [PATCH 1/4] HDFS-14487.Missing Space in Client Error Message Change-Id: I94ac5ebb220fb7e46d55bccf499c3e4a8f84e677 --- .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index a4e0742fa9e0c..4a0d75e2604fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -965,7 +965,7 @@ protected void completeFile(ExtendedBlock last) throws IOException { } try { if (retries == 0) { - throw new IOException("Unable to close file because the last block" + throw new IOException("Unable to close file because the last block " + last + " does not have enough number of replicas."); } retries--; From 4633312f6d97ff97ccaa6db7bcadedeee7bfef32 Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Wed, 22 May 2019 15:52:58 -0700 Subject: [PATCH 2/4] HDFS-14494. Move Server logging of StatedId inside receiveRequestState() Change-Id: I14b5a0c1683d0750130c9186c1b21bfdae531db3 --- .../src/main/java/org/apache/hadoop/ipc/Server.java | 2 -- .../hadoop/hdfs/server/namenode/GlobalStateIdContext.java | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 6fc907fb2e258..9018bed80e2f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -2676,8 +2676,6 @@ private void processRpcRequest(RpcRequestHeaderProto header, stateId = alignmentContext.receiveRequestState( header, getMaxIdleTime()); call.setClientStateId(stateId); - LOG.trace("Client State ID= {} and Server State ID= {}", - call.getClientStateId(), alignmentContext.getLastSeenStateId()); } } catch (IOException ioe) { throw new RpcServerException("Processing RPC request caught ", ioe); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java index 9f3a135336b9b..738d0a3ac942d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java @@ -124,9 +124,11 @@ public void updateRequestState(RpcRequestHeaderProto.Builder header) { @Override public long receiveRequestState(RpcRequestHeaderProto header, long clientWaitTime) throws RetriableException { - long serverStateId = - namesystem.getFSImage().getCorrectLastAppliedOrWrittenTxId(); + long serverStateId = getLastSeenStateId(); long clientStateId = header.getStateId(); + FSNamesystem.LOG.trace("Client State ID= {} and Server State ID= {}", + clientStateId, serverStateId); + if (clientStateId > serverStateId && HAServiceState.ACTIVE.equals(namesystem.getState())) { FSNamesystem.LOG.warn("The client stateId: {} is greater than " From 33daf390c9724dbc74c341293a07d8f3735b0654 Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Tue, 28 May 2019 10:30:02 -0700 Subject: [PATCH 3/4] HADOOP-16156. [Clean-up] Remove NULL check before instanceof and fix checkstyle in InnerNodeImpl Change-Id: Ie9d3866e88b067c55a955fbb005a8c3460517338 --- .../org/apache/hadoop/net/InnerNodeImpl.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java index 5a2931bf6a153..a0a977334f417 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java @@ -41,26 +41,29 @@ public InnerNodeImpl newInnerNode(String path) { protected final Map childrenMap = new HashMap<>(); protected int numOfLeaves; - /** Construct an InnerNode from a path-like string */ + /** Construct an InnerNode from a path-like string. */ protected InnerNodeImpl(String path) { super(path); } /** Construct an InnerNode - * from its name, its network location, its parent, and its level */ - protected InnerNodeImpl(String name, String location, InnerNode parent, int level) { + * from its name, its network location, its parent, and its level. */ + protected InnerNodeImpl(String name, String location, + InnerNode parent, int level) { super(name, location, parent, level); } @Override - public List getChildren() {return children;} + public List getChildren() { + return children; + } - /** @return the number of children this node has */ + /** @return the number of children this node has. */ int getNumOfChildren() { return children.size(); } - /** Judge if this node represents a rack + /** Judge if this node represents a rack. * @return true if it has no child or its children are not InnerNodes */ public boolean isRack() { @@ -76,7 +79,7 @@ public boolean isRack() { return true; } - /** Judge if this node is an ancestor of node n + /** Judge if this node is an ancestor of node n. * * @param n a node * @return true if this node is an ancestor of n @@ -87,7 +90,7 @@ public boolean isAncestor(Node n) { startsWith(getPath(this)+NodeBase.PATH_SEPARATOR_STR); } - /** Judge if this node is the parent of node n + /** Judge if this node is the parent of node n. * * @param n a node * @return true if this node is the parent of n @@ -107,8 +110,9 @@ public String getNextAncestorName(Node n) { name = name.substring(1); } int index=name.indexOf(PATH_SEPARATOR); - if (index !=-1) + if (index != -1) { name = name.substring(0, index); + } return name; } @@ -168,7 +172,8 @@ public boolean add(Node n) { * @see InnerNodeImpl(String, String, InnerNode, int) */ private InnerNodeImpl createParentNode(String parentName) { - return new InnerNodeImpl(parentName, getPath(this), this, this.getLevel()+1); + return new InnerNodeImpl(parentName, + getPath(this), this, this.getLevel() + 1); } @Override @@ -220,14 +225,16 @@ public boolean remove(Node n) { @Override public Node getLoc(String loc) { - if (loc == null || loc.length() == 0) return this; + if (loc == null || loc.length() == 0) { + return this; + } String[] path = loc.split(PATH_SEPARATOR_STR, 2); - Node childnode = childrenMap.get(path[0]); - if (childnode == null) return null; // non-existing node - if (path.length == 1) return childnode; - if (childnode instanceof InnerNode) { - return ((InnerNode)childnode).getLoc(path[1]); + Node childNode = childrenMap.get(path[0]); + if (childNode == null || path.length == 1) { + return childNode; + } else if (childNode instanceof InnerNode) { + return ((InnerNode)childNode).getLoc(path[1]); } else { return null; } @@ -237,11 +244,10 @@ public Node getLoc(String loc) { public Node getLeaf(int leafIndex, Node excludedNode) { int count=0; // check if the excluded node a leaf - boolean isLeaf = - excludedNode == null || !(excludedNode instanceof InnerNode); + boolean isLeaf = !(excludedNode instanceof InnerNode); // calculate the total number of excluded leaf nodes int numOfExcludedLeaves = - isLeaf ? 1 : ((InnerNode)excludedNode).getNumOfLeaves(); + isLeaf ? 1 : ((InnerNode)excludedNode).getNumOfLeaves(); if (isLeafParent()) { // children are leaves if (isLeaf) { // excluded node is a leaf node if (excludedNode != null && From a4ccaf862a0e69b0ad57881db405b59bccef665c Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Fri, 7 Jun 2019 16:06:45 -0700 Subject: [PATCH 4/4] HDDS-1651. Create a http.policy config for Ozone Change-Id: I0470558d64a7a194c34ed79a0448c7a22f8c6f01 --- .../java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 ++ .../common/src/main/resources/ozone-default.xml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 1463c43e830f3..cc6231726972a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -64,6 +64,8 @@ public final class OzoneConfigKeys { "dfs.container.ratis.ipc"; public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858; + public static final String OZONE_HTTP_POLICY = "ozone.http.policy"; + /** * When set to true, allocate a random free port for ozone container, so that * a mini cluster is able to launch multiple containers on a node. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 33f058486c7fe..755ff6025170d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2487,4 +2487,14 @@ The number of Recon Tasks that are waiting on updates from OM. + + ozone.http.policy + HTTP_ONLY + This configures the HTTP endpoint for Ozone daemons: + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + - HTTP_AND_HTTPS : Service is provided both on http and https + +