Skip to content

Commit e383a61

Browse files
chandrasekhar-188kstoty
chandrasekhar-188k
authored and
stoty
committed
HBASE-28627 REST ScannerModel doesn't support includeStartRow/includeStopRow (#6494)
includes HBASE-29125 and JSON addendum Signed-off-by: Istvan Toth <[email protected]> (cherry picked from commit e7b48e4)
1 parent 3e2e55f commit e383a61

File tree

9 files changed

+350
-9
lines changed

9 files changed

+350
-9
lines changed

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,7 @@ public interface Constants {
112112
/** Configuration parameter to set rest client socket timeout */
113113
String REST_CLIENT_SOCKET_TIMEOUT = "hbase.rest.client.socket.timeout";
114114
int DEFAULT_REST_CLIENT_SOCKET_TIMEOUT = 30 * 1000;
115+
116+
String SCAN_INCLUDE_START_ROW = "includeStartRow";
117+
String SCAN_INCLUDE_STOP_ROW = "includeStopRow";
115118
}

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,9 @@ Response update(final ScannerModel model, final boolean replace, final UriInfo u
110110
try {
111111
Filter filter = ScannerResultGenerator.buildFilterFromModel(model);
112112
String tableName = tableResource.getName();
113-
ScannerResultGenerator gen = new ScannerResultGenerator(tableName, spec, filter,
114-
model.getCaching(), model.getCacheBlocks());
113+
ScannerResultGenerator gen =
114+
new ScannerResultGenerator(tableName, spec, filter, model.getCaching(),
115+
model.getCacheBlocks(), model.isIncludeStartRow(), model.isIncludeStopRow());
115116
String id = gen.getID();
116117
ScannerInstanceResource instance =
117118
new ScannerInstanceResource(tableName, id, gen, model.getBatch());

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,20 @@ public ScannerResultGenerator(final String tableName, final RowSpec rowspec, fin
6363

6464
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
6565
final int caching, final boolean cacheBlocks) throws IllegalArgumentException, IOException {
66+
this(tableName, rowspec, filter, caching, cacheBlocks, true, false);
67+
}
68+
69+
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
70+
final int caching, final boolean cacheBlocks, boolean includeStartRow, boolean includeStopRow)
71+
throws IOException {
6672
Table table = RESTServlet.getInstance().getTable(tableName);
6773
try {
6874
Scan scan;
6975
if (rowspec.hasEndRow()) {
70-
scan = new Scan(rowspec.getStartRow(), rowspec.getEndRow());
76+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow)
77+
.withStopRow(rowspec.getEndRow(), includeStopRow);
7178
} else {
72-
scan = new Scan(rowspec.getStartRow());
79+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow);
7380
}
7481
if (rowspec.hasColumns()) {
7582
byte[][] columns = rowspec.getColumns();

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
133133
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
134134
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
135135
@QueryParam(Constants.FILTER) String paramFilter,
136-
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
136+
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
137+
@DefaultValue("true") @QueryParam(Constants.SCAN_INCLUDE_START_ROW) boolean includeStartRow,
138+
@DefaultValue("false") @QueryParam(Constants.SCAN_INCLUDE_STOP_ROW) boolean includeStopRow) {
137139
try {
138140
Filter prefixFilter = null;
139141
Scan tableScan = new Scan();
@@ -142,7 +144,7 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
142144
byte[] prefixBytes = Bytes.toBytes(prefix);
143145
prefixFilter = new PrefixFilter(Bytes.toBytes(prefix));
144146
if (startRow.isEmpty()) {
145-
tableScan.setStartRow(prefixBytes);
147+
tableScan.withStartRow(prefixBytes, includeStartRow);
146148
}
147149
}
148150
if (LOG.isTraceEnabled()) {
@@ -156,9 +158,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
156158
tableScan.setMaxVersions(maxVersions);
157159
tableScan.setTimeRange(startTime, endTime);
158160
if (!startRow.isEmpty()) {
159-
tableScan.setStartRow(Bytes.toBytes(startRow));
161+
tableScan.withStartRow(Bytes.toBytes(startRow), includeStartRow);
160162
}
161-
tableScan.setStopRow(Bytes.toBytes(endRow));
163+
tableScan.withStopRow(Bytes.toBytes(endRow), includeStopRow);
162164
for (String col : column) {
163165
byte[][] parts = CellUtil.parseColumn(Bytes.toBytes(col.trim()));
164166
if (parts.length == 1) {

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,41 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
120120
private List<String> labels = new ArrayList<>();
121121
private boolean cacheBlocks = true;
122122

123+
@JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = IncludeStartRowFilter.class)
124+
private boolean includeStartRow = true;
125+
126+
@JsonInclude(value = JsonInclude.Include.NON_DEFAULT)
127+
private boolean includeStopRow = false;
128+
129+
@XmlAttribute
130+
public boolean isIncludeStopRow() {
131+
return includeStopRow;
132+
}
133+
134+
public void setIncludeStopRow(boolean includeStopRow) {
135+
this.includeStopRow = includeStopRow;
136+
}
137+
138+
@XmlAttribute
139+
public boolean isIncludeStartRow() {
140+
return includeStartRow;
141+
}
142+
143+
public void setIncludeStartRow(boolean includeStartRow) {
144+
this.includeStartRow = includeStartRow;
145+
}
146+
147+
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
148+
value = { "EQ_CHECK_FOR_OPERAND_NOT_COMPATIBLE_WITH_THIS", "HE_EQUALS_NO_HASHCODE",
149+
"HE_EQUALS_USE_HASHCODE" },
150+
justification = "1.The supplied value from the JSON Value Filter is of Type Boolean, hence supressing the check, 2.hashCode method will not be invoked, hence supressing the check, 3.hashCode method will not be invoked, hence supressing the check")
151+
private static class IncludeStartRowFilter {
152+
@Override
153+
public boolean equals(Object value) {
154+
return Boolean.TRUE.equals(value);
155+
}
156+
}
157+
123158
/**
124159
* Implement lazily-instantiated singleton as per recipe here:
125160
* http://literatejava.com/jvm/fastest-threadsafe-singleton-jvm/
@@ -726,6 +761,8 @@ public static ScannerModel fromScan(Scan scan) throws Exception {
726761
model.addLabel(label);
727762
}
728763
}
764+
model.setIncludeStartRow(scan.includeStartRow());
765+
model.setIncludeStopRow(scan.includeStopRow());
729766
return model;
730767
}
731768

@@ -977,6 +1014,8 @@ public Message messageFromObject() {
9771014
builder.addLabels(label);
9781015
}
9791016
builder.setCacheBlocks(cacheBlocks);
1017+
builder.setIncludeStartRow(includeStartRow);
1018+
builder.setIncludeStopRow(includeStopRow);
9801019
return builder.build();
9811020
}
9821021

@@ -1020,6 +1059,12 @@ public ProtobufMessageHandler getObjectFromMessage(CodedInputStream cis) throws
10201059
if (builder.hasCacheBlocks()) {
10211060
this.cacheBlocks = builder.getCacheBlocks();
10221061
}
1062+
if (builder.hasIncludeStartRow()) {
1063+
this.includeStartRow = builder.getIncludeStartRow();
1064+
}
1065+
if (builder.hasIncludeStopRow()) {
1066+
this.includeStopRow = builder.getIncludeStopRow();
1067+
}
10231068
return this;
10241069
}
10251070

‎hbase-rest/src/main/protobuf/ScannerMessage.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,6 @@ message Scanner {
3030
optional int32 caching = 9; // specifies REST scanner caching
3131
repeated string labels = 10;
3232
optional bool cacheBlocks = 11; // server side block caching hint
33+
optional bool includeStartRow = 13;
34+
optional bool includeStopRow = 14;
3335
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,4 +392,151 @@ public void deleteNonExistent() throws IOException {
392392
Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
393393
assertEquals(404, response.getCode());
394394
}
395+
396+
@Test
397+
public void testScannerWithIncludeStartStopRowXML() throws IOException, JAXBException {
398+
// new scanner
399+
ScannerModel model = new ScannerModel();
400+
model.addColumn(Bytes.toBytes(COLUMN_1));
401+
model.setStartRow(Bytes.toBytes("aaa"));
402+
model.setEndRow(Bytes.toBytes("aae"));
403+
StringWriter writer = new StringWriter();
404+
marshaller.marshal(model, writer);
405+
byte[] body = Bytes.toBytes(writer.toString());
406+
407+
conf.set("hbase.rest.readonly", "false");
408+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
409+
assertEquals(201, response.getCode());
410+
String scannerURI = response.getLocation();
411+
assertNotNull(scannerURI);
412+
413+
// get a cell set
414+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
415+
assertEquals(200, response.getCode());
416+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
417+
CellSetModel cellSet =
418+
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
419+
420+
assertEquals(4, countCellSet(cellSet));
421+
422+
// test with include the start row false
423+
model.setIncludeStartRow(false);
424+
writer = new StringWriter();
425+
marshaller.marshal(model, writer);
426+
body = Bytes.toBytes(writer.toString());
427+
428+
conf.set("hbase.rest.readonly", "false");
429+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
430+
assertEquals(201, response.getCode());
431+
scannerURI = response.getLocation();
432+
assertNotNull(scannerURI);
433+
434+
// get a cell set
435+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
436+
assertEquals(200, response.getCode());
437+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
438+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
439+
440+
assertEquals(3, countCellSet(cellSet));
441+
442+
// test with include stop row true and start row false
443+
model.setIncludeStartRow(false);
444+
model.setIncludeStopRow(true);
445+
writer = new StringWriter();
446+
marshaller.marshal(model, writer);
447+
body = Bytes.toBytes(writer.toString());
448+
449+
conf.set("hbase.rest.readonly", "false");
450+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
451+
assertEquals(201, response.getCode());
452+
scannerURI = response.getLocation();
453+
assertNotNull(scannerURI);
454+
455+
// get a cell set
456+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
457+
assertEquals(200, response.getCode());
458+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
459+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
460+
461+
assertEquals(4, countCellSet(cellSet));
462+
463+
// test with including the start row true and stop row true
464+
model.setIncludeStartRow(true);
465+
model.setIncludeStopRow(true);
466+
writer = new StringWriter();
467+
marshaller.marshal(model, writer);
468+
body = Bytes.toBytes(writer.toString());
469+
470+
conf.set("hbase.rest.readonly", "false");
471+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
472+
assertEquals(201, response.getCode());
473+
scannerURI = response.getLocation();
474+
assertNotNull(scannerURI);
475+
476+
// get a cell set
477+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
478+
assertEquals(200, response.getCode());
479+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
480+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
481+
482+
assertEquals(5, countCellSet(cellSet));
483+
}
484+
485+
@Test
486+
public void testScannerWithIncludeStartStopRowPB() throws IOException {
487+
// new scanner
488+
ScannerModel model = new ScannerModel();
489+
model.addColumn(Bytes.toBytes(COLUMN_1));
490+
model.setStartRow(Bytes.toBytes("aaa"));
491+
model.setEndRow(Bytes.toBytes("aae"));
492+
493+
// test put operation is forbidden in read-only mode
494+
conf.set("hbase.rest.readonly", "false");
495+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
496+
model.createProtobufOutput());
497+
assertEquals(201, response.getCode());
498+
String scannerURI = response.getLocation();
499+
assertNotNull(scannerURI);
500+
501+
// get a cell set
502+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
503+
assertEquals(200, response.getCode());
504+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
505+
CellSetModel cellSet = new CellSetModel();
506+
cellSet.getObjectFromMessage(response.getBody());
507+
assertEquals(4, countCellSet(cellSet));
508+
509+
// test with include start row false
510+
model.setIncludeStartRow(false);
511+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
512+
model.createProtobufOutput());
513+
assertEquals(201, response.getCode());
514+
scannerURI = response.getLocation();
515+
assertNotNull(scannerURI);
516+
517+
// get a cell set
518+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
519+
assertEquals(200, response.getCode());
520+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
521+
cellSet = new CellSetModel();
522+
cellSet.getObjectFromMessage(response.getBody());
523+
assertEquals(3, countCellSet(cellSet));
524+
525+
// test with include stop row true
526+
model.setIncludeStartRow(true);
527+
model.setIncludeStopRow(true);
528+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
529+
model.createProtobufOutput());
530+
assertEquals(201, response.getCode());
531+
scannerURI = response.getLocation();
532+
assertNotNull(scannerURI);
533+
534+
// get a cell set
535+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
536+
assertEquals(200, response.getCode());
537+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
538+
cellSet = new CellSetModel();
539+
cellSet.getObjectFromMessage(response.getBody());
540+
assertEquals(5, countCellSet(cellSet));
541+
}
395542
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,4 +630,82 @@ public void testLongLivedScan() throws Exception {
630630
Thread.sleep(trialPause);
631631
}
632632
}
633+
634+
@Test
635+
public void testScanWithInlcudeStartStopRow() throws Exception {
636+
int numTrials = 6;
637+
638+
// Truncate the test table for inserting test scenarios rows keys
639+
TEST_UTIL.getAdmin().disableTable(TABLE);
640+
TEST_UTIL.getAdmin().truncateTable(TABLE, false);
641+
String row = "testrow";
642+
643+
try (Table table = TEST_UTIL.getConnection().getTable(TABLE)) {
644+
List<Put> puts = new ArrayList<>();
645+
Put put = null;
646+
for (int i = 1; i <= numTrials; i++) {
647+
put = new Put(Bytes.toBytes(row + i));
648+
put.addColumn(COLUMN_1, QUALIFIER_1, TS_2, Bytes.toBytes("testvalue" + i));
649+
puts.add(put);
650+
}
651+
table.put(puts);
652+
}
653+
654+
remoteTable =
655+
new RemoteHTable(new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())),
656+
TEST_UTIL.getConfiguration(), TABLE.toBytes());
657+
658+
Scan scan =
659+
new Scan().withStartRow(Bytes.toBytes(row + "1")).withStopRow(Bytes.toBytes(row + "5"));
660+
661+
ResultScanner scanner = remoteTable.getScanner(scan);
662+
Iterator<Result> resultIterator = scanner.iterator();
663+
int counter = 0;
664+
while (resultIterator.hasNext()) {
665+
byte[] row1 = resultIterator.next().getRow();
666+
System.out.println(Bytes.toString(row1));
667+
counter++;
668+
}
669+
assertEquals(4, counter);
670+
671+
// test with include start row false
672+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
673+
.withStopRow(Bytes.toBytes(row + "5"));
674+
scanner = remoteTable.getScanner(scan);
675+
resultIterator = scanner.iterator();
676+
counter = 0;
677+
while (resultIterator.hasNext()) {
678+
byte[] row1 = resultIterator.next().getRow();
679+
System.out.println(Bytes.toString(row1));
680+
counter++;
681+
}
682+
assertEquals(3, counter);
683+
684+
// test with include start row false and stop row true
685+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
686+
.withStopRow(Bytes.toBytes(row + "5"), true);
687+
scanner = remoteTable.getScanner(scan);
688+
resultIterator = scanner.iterator();
689+
counter = 0;
690+
while (resultIterator.hasNext()) {
691+
byte[] row1 = resultIterator.next().getRow();
692+
System.out.println(Bytes.toString(row1));
693+
counter++;
694+
}
695+
assertEquals(4, counter);
696+
697+
// test with include start row true and stop row true
698+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), true)
699+
.withStopRow(Bytes.toBytes(row + "5"), true);
700+
scanner = remoteTable.getScanner(scan);
701+
resultIterator = scanner.iterator();
702+
counter = 0;
703+
while (resultIterator.hasNext()) {
704+
byte[] row1 = resultIterator.next().getRow();
705+
System.out.println(Bytes.toString(row1));
706+
counter++;
707+
}
708+
assertEquals(5, counter);
709+
}
710+
633711
}

0 commit comments

Comments
 (0)