Skip to content

Commit 4fc4bfa

Browse files
tomscuttomscut
committed
support json format
1 parent 6a97f6a commit 4fc4bfa

File tree

2 files changed

+166
-5
lines changed

2 files changed

+166
-5
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NetworkTopologyServlet.java‎

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,20 @@
1717
*/
1818
package org.apache.hadoop.hdfs.server.namenode;
1919

20+
import com.fasterxml.jackson.core.JsonFactory;
21+
import com.fasterxml.jackson.core.JsonGenerator;
2022
import org.apache.hadoop.classification.InterfaceAudience;
2123
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
2224
import org.apache.hadoop.net.NetUtils;
2325
import org.apache.hadoop.net.Node;
2426
import org.apache.hadoop.net.NodeBase;
27+
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
2528
import org.apache.hadoop.util.StringUtils;
2629

2730
import javax.servlet.ServletContext;
2831
import javax.servlet.http.HttpServletRequest;
2932
import javax.servlet.http.HttpServletResponse;
33+
import javax.ws.rs.core.HttpHeaders;
3034
import java.io.IOException;
3135
import java.io.PrintStream;
3236
import java.util.ArrayList;
@@ -44,19 +48,29 @@ public class NetworkTopologyServlet extends DfsServlet {
4448

4549
public static final String PATH_SPEC = "/topology";
4650

51+
protected static final String FORMAT_JSON = "json";
52+
protected static final String FORMAT_TEXT = "text";
53+
4754
@Override
4855
public void doGet(HttpServletRequest request, HttpServletResponse response)
4956
throws IOException {
5057
final ServletContext context = getServletContext();
58+
59+
String format = parseAcceptHeader(request);
60+
if (FORMAT_TEXT.equals(format)) {
61+
response.setContentType("text/plain; charset=UTF-8");
62+
} else if (FORMAT_JSON.equals(format)) {
63+
response.setContentType("application/json; charset=UTF-8");
64+
}
65+
5166
NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
5267
BlockManager bm = nn.getNamesystem().getBlockManager();
5368
List<Node> leaves = bm.getDatanodeManager().getNetworkTopology()
5469
.getLeaves(NodeBase.ROOT);
5570

56-
response.setContentType("text/plain; charset=UTF-8");
5771
try (PrintStream out = new PrintStream(
5872
response.getOutputStream(), false, "UTF-8")) {
59-
printTopology(out, leaves);
73+
printTopology(out, leaves, format);
6074
} catch (Throwable t) {
6175
String errMsg = "Print network topology failed. "
6276
+ StringUtils.stringifyException(t);
@@ -74,8 +88,10 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
7488
*
7589
* @param stream print stream
7690
* @param leaves leaves nodes under base scope
91+
* @param format the response format
7792
*/
78-
public void printTopology(PrintStream stream, List<Node> leaves) {
93+
public void printTopology(PrintStream stream, List<Node> leaves,
94+
String format) throws BadFormatException, IOException {
7995
if (leaves.isEmpty()) {
8096
stream.print("No DataNodes");
8197
return;
@@ -95,6 +111,49 @@ public void printTopology(PrintStream stream, List<Node> leaves) {
95111
ArrayList<String> racks = new ArrayList<>(tree.keySet());
96112
Collections.sort(racks);
97113

114+
if (FORMAT_JSON.equals(format)) {
115+
printJsonFormat(stream, tree, racks);
116+
} else if (FORMAT_TEXT.equals(format)) {
117+
printTextFormat(stream, tree, racks);
118+
} else {
119+
throw new BadFormatException("Bad format: " + format);
120+
}
121+
}
122+
123+
private void printJsonFormat(PrintStream stream, Map<String,
124+
TreeSet<String>> tree, ArrayList<String> racks) throws IOException {
125+
JsonFactory dumpFactory = new JsonFactory();
126+
JsonGenerator dumpGenerator = dumpFactory.createGenerator(stream);
127+
dumpGenerator.writeStartArray();
128+
129+
for(String r : racks) {
130+
dumpGenerator.writeStartObject();
131+
dumpGenerator.writeFieldName(r);
132+
TreeSet<String> nodes = tree.get(r);
133+
dumpGenerator.writeStartArray();
134+
135+
for(String n : nodes) {
136+
dumpGenerator.writeStartObject();
137+
dumpGenerator.writeStringField("ip", n);
138+
String hostname = NetUtils.getHostNameOfIP(n);
139+
if(hostname != null) {
140+
dumpGenerator.writeStringField("hostname", hostname);
141+
}
142+
dumpGenerator.writeEndObject();
143+
}
144+
dumpGenerator.writeEndArray();
145+
dumpGenerator.writeEndObject();
146+
}
147+
dumpGenerator.writeEndArray();
148+
dumpGenerator.flush();
149+
150+
if (!dumpGenerator.isClosed()) {
151+
dumpGenerator.close();
152+
}
153+
}
154+
155+
private void printTextFormat(PrintStream stream, Map<String,
156+
TreeSet<String>> tree, ArrayList<String> racks) {
98157
for(String r : racks) {
99158
stream.println("Rack: " + r);
100159
TreeSet<String> nodes = tree.get(r);
@@ -110,4 +169,19 @@ public void printTopology(PrintStream stream, List<Node> leaves) {
110169
stream.println();
111170
}
112171
}
172+
173+
@VisibleForTesting
174+
static String parseAcceptHeader(HttpServletRequest request) {
175+
String format = request.getHeader(HttpHeaders.ACCEPT);
176+
return format != null && format.contains(FORMAT_JSON) ?
177+
FORMAT_JSON : FORMAT_TEXT;
178+
}
179+
180+
public static class BadFormatException extends Exception {
181+
private static final long serialVersionUID = 1L;
182+
183+
public BadFormatException(String msg) {
184+
super(msg);
185+
}
186+
}
113187
}

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
*/
1818
package org.apache.hadoop.hdfs.server.namenode;
1919

20+
import com.fasterxml.jackson.databind.JsonNode;
21+
import com.fasterxml.jackson.databind.ObjectMapper;
2022
import org.apache.hadoop.conf.Configuration;
2123
import org.apache.hadoop.hdfs.HdfsConfiguration;
2224
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -29,14 +31,16 @@
2931
import java.net.HttpURLConnection;
3032
import java.net.URL;
3133
import java.util.ArrayList;
34+
import java.util.Iterator;
35+
import java.util.Map;
3236

3337
import static org.junit.Assert.assertEquals;
3438
import static org.junit.Assert.assertTrue;
3539

3640
public class TestNetworkTopologyServlet {
3741

3842
@Test
39-
public void testPrintTopology() throws IOException {
43+
public void testPrintTopologyTextFormat() throws IOException {
4044
StaticMapping.resetMap();
4145
Configuration conf = new HdfsConfiguration();
4246
int dataNodesNum = 0;
@@ -84,7 +88,59 @@ public void testPrintTopology() throws IOException {
8488
}
8589

8690
@Test
87-
public void testPrintTopologyNoDatanodes() throws IOException {
91+
public void testPrintTopologyJsonFormat() throws IOException {
92+
StaticMapping.resetMap();
93+
Configuration conf = new HdfsConfiguration();
94+
int dataNodesNum = 0;
95+
final ArrayList<String> rackList = new ArrayList<String>();
96+
for (int i = 0; i < 5; i++) {
97+
for (int j = 0; j < 2; j++) {
98+
rackList.add("/rack" + i);
99+
dataNodesNum++;
100+
}
101+
}
102+
103+
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
104+
.numDataNodes(dataNodesNum)
105+
.racks(rackList.toArray(new String[rackList.size()]))
106+
.build();
107+
cluster.waitActive();
108+
109+
// get http uri
110+
String httpUri = cluster.getHttpUri(0);
111+
112+
// send http request
113+
URL url = new URL(httpUri + "/topology");
114+
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
115+
conn.setReadTimeout(20000);
116+
conn.setConnectTimeout(20000);
117+
conn.setRequestProperty("Accept", "application/json");
118+
conn.connect();
119+
ByteArrayOutputStream out = new ByteArrayOutputStream();
120+
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
121+
String topology = out.toString();
122+
123+
// parse json
124+
JsonNode racks = new ObjectMapper().readTree(topology);
125+
126+
// assert rack number
127+
assertEquals(racks.size(), 5);
128+
129+
// assert node number
130+
Iterator<JsonNode> elements = racks.elements();
131+
int dataNodesCount = 0;
132+
while(elements.hasNext()){
133+
JsonNode rack = elements.next();
134+
Iterator<Map.Entry<String, JsonNode>> fields = rack.fields();
135+
while (fields.hasNext()) {
136+
dataNodesCount += fields.next().getValue().size();
137+
}
138+
}
139+
assertEquals(dataNodesCount, dataNodesNum);
140+
}
141+
142+
@Test
143+
public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
88144
StaticMapping.resetMap();
89145
Configuration conf = new HdfsConfiguration();
90146
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
@@ -112,4 +168,35 @@ public void testPrintTopologyNoDatanodes() throws IOException {
112168
// assert node number
113169
assertTrue(topology.contains("No DataNodes"));
114170
}
171+
172+
@Test
173+
public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
174+
StaticMapping.resetMap();
175+
Configuration conf = new HdfsConfiguration();
176+
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
177+
.numDataNodes(0)
178+
.build();
179+
cluster.waitActive();
180+
181+
// get http uri
182+
String httpUri = cluster.getHttpUri(0);
183+
184+
// send http request
185+
URL url = new URL(httpUri + "/topology");
186+
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
187+
conn.setReadTimeout(20000);
188+
conn.setConnectTimeout(20000);
189+
conn.setRequestProperty("Accept", "application/json");
190+
conn.connect();
191+
ByteArrayOutputStream out = new ByteArrayOutputStream();
192+
IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
193+
StringBuilder sb =
194+
new StringBuilder("-- Network Topology -- \n");
195+
sb.append(out);
196+
sb.append("\n-- Network Topology -- ");
197+
String topology = sb.toString();
198+
199+
// assert node number
200+
assertTrue(topology.contains("No DataNodes"));
201+
}
115202
}

0 commit comments

Comments
 (0)