Skip to content

Commit d0a0b8e

Browse files
chandrasekhar-188kchandrasekhar-188k
committed
HBASE-28627 REST ScannerModel doesn't support includeStartRow/includeStopRow
1 parent 20e510f commit d0a0b8e

File tree

9 files changed

+349
-9
lines changed

9 files changed

+349
-9
lines changed

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,7 @@ public interface Constants {
112112
/** Configuration parameter to set rest client socket timeout */
113113
String REST_CLIENT_SOCKET_TIMEOUT = "hbase.rest.client.socket.timeout";
114114
int DEFAULT_REST_CLIENT_SOCKET_TIMEOUT = 30 * 1000;
115+
116+
String SCAN_INCLUDE_START_ROW = "includeStartRow";
117+
String SCAN_INCLUDE_STOP_ROW = "includeStopRow";
115118
}

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,9 @@ Response update(final ScannerModel model, final boolean replace, final UriInfo u
110110
try {
111111
Filter filter = ScannerResultGenerator.buildFilterFromModel(model);
112112
String tableName = tableResource.getName();
113-
ScannerResultGenerator gen = new ScannerResultGenerator(tableName, spec, filter,
114-
model.getCaching(), model.getCacheBlocks());
113+
ScannerResultGenerator gen =
114+
new ScannerResultGenerator(tableName, spec, filter, model.getCaching(),
115+
model.getCacheBlocks(), model.isIncludeStartRow(), model.isIncludeStopRow());
115116
String id = gen.getID();
116117
ScannerInstanceResource instance =
117118
new ScannerInstanceResource(tableName, id, gen, model.getBatch());

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,20 @@ public ScannerResultGenerator(final String tableName, final RowSpec rowspec, fin
6363

6464
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
6565
final int caching, final boolean cacheBlocks) throws IllegalArgumentException, IOException {
66+
this(tableName, rowspec, filter, caching, cacheBlocks, true, false);
67+
}
68+
69+
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
70+
final int caching, final boolean cacheBlocks, boolean includeStartRow, boolean includeStopRow)
71+
throws IOException {
6672
Table table = RESTServlet.getInstance().getTable(tableName);
6773
try {
6874
Scan scan;
6975
if (rowspec.hasEndRow()) {
70-
scan = new Scan(rowspec.getStartRow(), rowspec.getEndRow());
76+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow)
77+
.withStopRow(rowspec.getEndRow(), includeStopRow);
7178
} else {
72-
scan = new Scan(rowspec.getStartRow());
79+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow);
7380
}
7481
if (rowspec.hasColumns()) {
7582
byte[][] columns = rowspec.getColumns();

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
135135
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
136136
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
137137
@QueryParam(Constants.FILTER) String paramFilter,
138-
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
138+
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
139+
@DefaultValue("true") @QueryParam(Constants.SCAN_INCLUDE_START_ROW) boolean includeStartRow,
140+
@DefaultValue("false") @QueryParam(Constants.SCAN_INCLUDE_STOP_ROW) boolean includeStopRow) {
139141
try {
140142
Filter prefixFilter = null;
141143
Scan tableScan = new Scan();
@@ -144,7 +146,7 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
144146
byte[] prefixBytes = Bytes.toBytes(prefix);
145147
prefixFilter = new PrefixFilter(Bytes.toBytes(prefix));
146148
if (startRow.isEmpty()) {
147-
tableScan.setStartRow(prefixBytes);
149+
tableScan.withStartRow(prefixBytes, includeStartRow);
148150
}
149151
}
150152
if (LOG.isTraceEnabled()) {
@@ -158,9 +160,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
158160
tableScan.setMaxVersions(maxVersions);
159161
tableScan.setTimeRange(startTime, endTime);
160162
if (!startRow.isEmpty()) {
161-
tableScan.setStartRow(Bytes.toBytes(startRow));
163+
tableScan.withStartRow(Bytes.toBytes(startRow), includeStartRow);
162164
}
163-
tableScan.setStopRow(Bytes.toBytes(endRow));
165+
tableScan.withStopRow(Bytes.toBytes(endRow), includeStopRow);
164166
for (String col : column) {
165167
byte[][] parts = CellUtil.parseColumn(Bytes.toBytes(col.trim()));
166168
if (parts.length == 1) {

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,40 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
120120
private List<String> labels = new ArrayList<>();
121121
private boolean cacheBlocks = true;
122122

123+
@JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = IncludeStartRowFilter.class)
124+
private boolean includeStartRow = true;
125+
126+
@JsonInclude(value = JsonInclude.Include.NON_DEFAULT)
127+
private boolean includeStopRow = false;
128+
129+
@XmlAttribute
130+
public boolean isIncludeStopRow() {
131+
return includeStopRow;
132+
}
133+
134+
public void setIncludeStopRow(boolean includeStopRow) {
135+
this.includeStopRow = includeStopRow;
136+
}
137+
138+
@XmlAttribute
139+
public boolean isIncludeStartRow() {
140+
return includeStartRow;
141+
}
142+
143+
public void setIncludeStartRow(boolean includeStartRow) {
144+
this.includeStartRow = includeStartRow;
145+
}
146+
147+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
148+
value = { "EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS", "HE_EQUALS_NO_HASHCODE" },
149+
justification = "1.The supplied value from the JSON Value Filter is of Type Boolean, hence supressing the check, 2.hashCode method will not be invoked, hence supressing the check")
150+
private static class IncludeStartRowFilter {
151+
@Override
152+
public boolean equals(Object value) {
153+
return Boolean.TRUE.equals(value);
154+
}
155+
}
156+
123157
/**
124158
* Implement lazily-instantiated singleton as per recipe here:
125159
* http://literatejava.com/jvm/fastest-threadsafe-singleton-jvm/
@@ -726,6 +760,8 @@ public static ScannerModel fromScan(Scan scan) throws Exception {
726760
model.addLabel(label);
727761
}
728762
}
763+
model.setIncludeStartRow(scan.includeStartRow());
764+
model.setIncludeStopRow(scan.includeStopRow());
729765
return model;
730766
}
731767

@@ -977,6 +1013,8 @@ public Message messageFromObject() {
9771013
builder.addLabels(label);
9781014
}
9791015
builder.setCacheBlocks(cacheBlocks);
1016+
builder.setIncludeStartRow(includeStartRow);
1017+
builder.setIncludeStopRow(includeStopRow);
9801018
return builder.build();
9811019
}
9821020

@@ -1020,6 +1058,12 @@ public ProtobufMessageHandler getObjectFromMessage(CodedInputStream cis) throws
10201058
if (builder.hasCacheBlocks()) {
10211059
this.cacheBlocks = builder.getCacheBlocks();
10221060
}
1061+
if (builder.hasIncludeStartRow()) {
1062+
this.includeStartRow = builder.getIncludeStartRow();
1063+
}
1064+
if (builder.hasIncludeStopRow()) {
1065+
this.includeStopRow = builder.getIncludeStopRow();
1066+
}
10231067
return this;
10241068
}
10251069

‎hbase-rest/src/main/protobuf/ScannerMessage.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,6 @@ message Scanner {
3030
optional int32 caching = 9; // specifies REST scanner caching
3131
repeated string labels = 10;
3232
optional bool cacheBlocks = 11; // server side block caching hint
33+
optional bool includeStartRow = 13;
34+
optional bool includeStopRow = 14;
3335
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,4 +392,151 @@ public void deleteNonExistent() throws IOException {
392392
Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
393393
assertEquals(404, response.getCode());
394394
}
395+
396+
@Test
397+
public void testScannerWithIncludeStartStopRowXML() throws IOException, JAXBException {
398+
// new scanner
399+
ScannerModel model = new ScannerModel();
400+
model.addColumn(Bytes.toBytes(COLUMN_1));
401+
model.setStartRow(Bytes.toBytes("aaa"));
402+
model.setEndRow(Bytes.toBytes("aae"));
403+
StringWriter writer = new StringWriter();
404+
marshaller.marshal(model, writer);
405+
byte[] body = Bytes.toBytes(writer.toString());
406+
407+
conf.set("hbase.rest.readonly", "false");
408+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
409+
assertEquals(201, response.getCode());
410+
String scannerURI = response.getLocation();
411+
assertNotNull(scannerURI);
412+
413+
// get a cell set
414+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
415+
assertEquals(200, response.getCode());
416+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
417+
CellSetModel cellSet =
418+
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
419+
420+
assertEquals(4, countCellSet(cellSet));
421+
422+
// test with include the start row false
423+
model.setIncludeStartRow(false);
424+
writer = new StringWriter();
425+
marshaller.marshal(model, writer);
426+
body = Bytes.toBytes(writer.toString());
427+
428+
conf.set("hbase.rest.readonly", "false");
429+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
430+
assertEquals(201, response.getCode());
431+
scannerURI = response.getLocation();
432+
assertNotNull(scannerURI);
433+
434+
// get a cell set
435+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
436+
assertEquals(200, response.getCode());
437+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
438+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
439+
440+
assertEquals(3, countCellSet(cellSet));
441+
442+
// test with include stop row true and start row false
443+
model.setIncludeStartRow(false);
444+
model.setIncludeStopRow(true);
445+
writer = new StringWriter();
446+
marshaller.marshal(model, writer);
447+
body = Bytes.toBytes(writer.toString());
448+
449+
conf.set("hbase.rest.readonly", "false");
450+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
451+
assertEquals(201, response.getCode());
452+
scannerURI = response.getLocation();
453+
assertNotNull(scannerURI);
454+
455+
// get a cell set
456+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
457+
assertEquals(200, response.getCode());
458+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
459+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
460+
461+
assertEquals(4, countCellSet(cellSet));
462+
463+
// test with including the start row true and stop row true
464+
model.setIncludeStartRow(true);
465+
model.setIncludeStopRow(true);
466+
writer = new StringWriter();
467+
marshaller.marshal(model, writer);
468+
body = Bytes.toBytes(writer.toString());
469+
470+
conf.set("hbase.rest.readonly", "false");
471+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
472+
assertEquals(201, response.getCode());
473+
scannerURI = response.getLocation();
474+
assertNotNull(scannerURI);
475+
476+
// get a cell set
477+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
478+
assertEquals(200, response.getCode());
479+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
480+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
481+
482+
assertEquals(5, countCellSet(cellSet));
483+
}
484+
485+
@Test
486+
public void testScannerWithIncludeStartStopRowPB() throws IOException {
487+
// new scanner
488+
ScannerModel model = new ScannerModel();
489+
model.addColumn(Bytes.toBytes(COLUMN_1));
490+
model.setStartRow(Bytes.toBytes("aaa"));
491+
model.setEndRow(Bytes.toBytes("aae"));
492+
493+
// test put operation is forbidden in read-only mode
494+
conf.set("hbase.rest.readonly", "false");
495+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
496+
model.createProtobufOutput());
497+
assertEquals(201, response.getCode());
498+
String scannerURI = response.getLocation();
499+
assertNotNull(scannerURI);
500+
501+
// get a cell set
502+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
503+
assertEquals(200, response.getCode());
504+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
505+
CellSetModel cellSet = new CellSetModel();
506+
cellSet.getObjectFromMessage(response.getBody());
507+
assertEquals(4, countCellSet(cellSet));
508+
509+
// test with include start row false
510+
model.setIncludeStartRow(false);
511+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
512+
model.createProtobufOutput());
513+
assertEquals(201, response.getCode());
514+
scannerURI = response.getLocation();
515+
assertNotNull(scannerURI);
516+
517+
// get a cell set
518+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
519+
assertEquals(200, response.getCode());
520+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
521+
cellSet = new CellSetModel();
522+
cellSet.getObjectFromMessage(response.getBody());
523+
assertEquals(3, countCellSet(cellSet));
524+
525+
// test with include stop row true
526+
model.setIncludeStartRow(true);
527+
model.setIncludeStopRow(true);
528+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
529+
model.createProtobufOutput());
530+
assertEquals(201, response.getCode());
531+
scannerURI = response.getLocation();
532+
assertNotNull(scannerURI);
533+
534+
// get a cell set
535+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
536+
assertEquals(200, response.getCode());
537+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
538+
cellSet = new CellSetModel();
539+
cellSet.getObjectFromMessage(response.getBody());
540+
assertEquals(5, countCellSet(cellSet));
541+
}
395542
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,4 +630,82 @@ public void testLongLivedScan() throws Exception {
630630
Thread.sleep(trialPause);
631631
}
632632
}
633+
634+
@Test
635+
public void testScanWithInlcudeStartStopRow() throws Exception {
636+
int numTrials = 6;
637+
638+
// Truncate the test table for inserting test scenarios rows keys
639+
TEST_UTIL.getAdmin().disableTable(TABLE);
640+
TEST_UTIL.getAdmin().truncateTable(TABLE, false);
641+
String row = "testrow";
642+
643+
try (Table table = TEST_UTIL.getConnection().getTable(TABLE)) {
644+
List<Put> puts = new ArrayList<>();
645+
Put put = null;
646+
for (int i = 1; i <= numTrials; i++) {
647+
put = new Put(Bytes.toBytes(row + i));
648+
put.addColumn(COLUMN_1, QUALIFIER_1, TS_2, Bytes.toBytes("testvalue" + i));
649+
puts.add(put);
650+
}
651+
table.put(puts);
652+
}
653+
654+
remoteTable =
655+
new RemoteHTable(new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())),
656+
TEST_UTIL.getConfiguration(), TABLE.toBytes());
657+
658+
Scan scan =
659+
new Scan().withStartRow(Bytes.toBytes(row + "1")).withStopRow(Bytes.toBytes(row + "5"));
660+
661+
ResultScanner scanner = remoteTable.getScanner(scan);
662+
Iterator<Result> resultIterator = scanner.iterator();
663+
int counter = 0;
664+
while (resultIterator.hasNext()) {
665+
byte[] row1 = resultIterator.next().getRow();
666+
System.out.println(Bytes.toString(row1));
667+
counter++;
668+
}
669+
assertEquals(4, counter);
670+
671+
// test with include start row false
672+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
673+
.withStopRow(Bytes.toBytes(row + "5"));
674+
scanner = remoteTable.getScanner(scan);
675+
resultIterator = scanner.iterator();
676+
counter = 0;
677+
while (resultIterator.hasNext()) {
678+
byte[] row1 = resultIterator.next().getRow();
679+
System.out.println(Bytes.toString(row1));
680+
counter++;
681+
}
682+
assertEquals(3, counter);
683+
684+
// test with include start row false and stop row true
685+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
686+
.withStopRow(Bytes.toBytes(row + "5"), true);
687+
scanner = remoteTable.getScanner(scan);
688+
resultIterator = scanner.iterator();
689+
counter = 0;
690+
while (resultIterator.hasNext()) {
691+
byte[] row1 = resultIterator.next().getRow();
692+
System.out.println(Bytes.toString(row1));
693+
counter++;
694+
}
695+
assertEquals(4, counter);
696+
697+
// test with include start row true and stop row true
698+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), true)
699+
.withStopRow(Bytes.toBytes(row + "5"), true);
700+
scanner = remoteTable.getScanner(scan);
701+
resultIterator = scanner.iterator();
702+
counter = 0;
703+
while (resultIterator.hasNext()) {
704+
byte[] row1 = resultIterator.next().getRow();
705+
System.out.println(Bytes.toString(row1));
706+
counter++;
707+
}
708+
assertEquals(5, counter);
709+
}
710+
633711
}

0 commit comments

Comments
 (0)