Skip to content

Commit f923da5

Browse files
chandrasekhar-188kPrathyusha Garre
chandrasekhar-188k
authored and
Prathyusha Garre
committed
HBASE-28627 REST ScannerModel doesn't support includeStartRow/includeStopRow (apache#6374)
Signed-off-by: Istvan Toth <[email protected]>
1 parent b7748ff commit f923da5

File tree

9 files changed

+282
-10
lines changed

9 files changed

+282
-10
lines changed

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,7 @@ public interface Constants {
112112
/** Configuration parameter to set rest client socket timeout */
113113
String REST_CLIENT_SOCKET_TIMEOUT = "hbase.rest.client.socket.timeout";
114114
int DEFAULT_REST_CLIENT_SOCKET_TIMEOUT = 30 * 1000;
115+
116+
String SCAN_INCLUDE_START_ROW = "includeStartRow";
117+
String SCAN_INCLUDE_STOP_ROW = "includeStopRow";
115118
}

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ Response update(final ScannerModel model, final boolean replace, final UriInfo u
111111
Filter filter = ScannerResultGenerator.buildFilterFromModel(model);
112112
String tableName = tableResource.getName();
113113
ScannerResultGenerator gen = new ScannerResultGenerator(tableName, spec, filter,
114-
model.getCaching(), model.getCacheBlocks(), model.getLimit());
114+
model.getCaching(), model.getCacheBlocks(), model.getLimit(), model.isIncludeStartRow(),
115+
model.isIncludeStopRow());
115116
String id = gen.getID();
116117
ScannerInstanceResource instance =
117118
new ScannerInstanceResource(tableName, id, gen, model.getBatch());

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,20 @@ public ScannerResultGenerator(final String tableName, final RowSpec rowspec, fin
6363

6464
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
6565
final int caching, final boolean cacheBlocks) throws IllegalArgumentException, IOException {
66-
this(tableName, rowspec, filter, caching, cacheBlocks, -1);
66+
this(tableName, rowspec, filter, caching, cacheBlocks, -1, true, false);
6767
}
6868

6969
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
70-
final int caching, final boolean cacheBlocks, int limit) throws IOException {
70+
final int caching, final boolean cacheBlocks, int limit, boolean includeStartRow,
71+
boolean includeStopRow) throws IOException {
7172
Table table = RESTServlet.getInstance().getTable(tableName);
7273
try {
7374
Scan scan;
7475
if (rowspec.hasEndRow()) {
75-
scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
76+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow)
77+
.withStopRow(rowspec.getEndRow(), includeStopRow);
7678
} else {
77-
scan = new Scan().withStartRow(rowspec.getStartRow());
79+
scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow);
7880
}
7981
if (rowspec.hasColumns()) {
8082
byte[][] columns = rowspec.getColumns();

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
135135
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
136136
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
137137
@QueryParam(Constants.FILTER) String paramFilter,
138-
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
138+
@QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
139+
@DefaultValue("true") @QueryParam(Constants.SCAN_INCLUDE_START_ROW) boolean includeStartRow,
140+
@DefaultValue("false") @QueryParam(Constants.SCAN_INCLUDE_STOP_ROW) boolean includeStopRow) {
139141
try {
140142
Filter prefixFilter = null;
141143
Scan tableScan = new Scan();
@@ -144,7 +146,7 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
144146
byte[] prefixBytes = Bytes.toBytes(prefix);
145147
prefixFilter = new PrefixFilter(Bytes.toBytes(prefix));
146148
if (startRow.isEmpty()) {
147-
tableScan.withStartRow(prefixBytes);
149+
tableScan.withStartRow(prefixBytes, includeStartRow);
148150
}
149151
}
150152
if (LOG.isTraceEnabled()) {
@@ -158,9 +160,9 @@ public TableScanResource getScanResource(final @PathParam("scanspec") String sca
158160
tableScan.readVersions(maxVersions);
159161
tableScan.setTimeRange(startTime, endTime);
160162
if (!startRow.isEmpty()) {
161-
tableScan.withStartRow(Bytes.toBytes(startRow));
163+
tableScan.withStartRow(Bytes.toBytes(startRow), includeStartRow);
162164
}
163-
tableScan.withStopRow(Bytes.toBytes(endRow));
165+
tableScan.withStopRow(Bytes.toBytes(endRow), includeStopRow);
164166
for (String col : column) {
165167
byte[][] parts = CellUtil.parseColumn(Bytes.toBytes(col.trim()));
166168
if (parts.length == 1) {

‎hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,28 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
121121
private boolean cacheBlocks = true;
122122
private int limit = -1;
123123

124+
private boolean includeStartRow = true;
125+
126+
private boolean includeStopRow = false;
127+
128+
@XmlAttribute
129+
public boolean isIncludeStopRow() {
130+
return includeStopRow;
131+
}
132+
133+
public void setIncludeStopRow(boolean includeStopRow) {
134+
this.includeStopRow = includeStopRow;
135+
}
136+
137+
@XmlAttribute
138+
public boolean isIncludeStartRow() {
139+
return includeStartRow;
140+
}
141+
142+
public void setIncludeStartRow(boolean includeStartRow) {
143+
this.includeStartRow = includeStartRow;
144+
}
145+
124146
/**
125147
* Implement lazily-instantiated singleton as per recipe here:
126148
* http://literatejava.com/jvm/fastest-threadsafe-singleton-jvm/
@@ -730,6 +752,8 @@ public static ScannerModel fromScan(Scan scan) throws Exception {
730752
model.addLabel(label);
731753
}
732754
}
755+
model.setIncludeStartRow(scan.includeStartRow());
756+
model.setIncludeStopRow(scan.includeStopRow());
733757
return model;
734758
}
735759

@@ -997,6 +1021,8 @@ public Message messageFromObject() {
9971021
builder.addLabels(label);
9981022
}
9991023
builder.setCacheBlocks(cacheBlocks);
1024+
builder.setIncludeStartRow(includeStartRow);
1025+
builder.setIncludeStopRow(includeStopRow);
10001026
return builder.build();
10011027
}
10021028

@@ -1043,6 +1069,12 @@ public ProtobufMessageHandler getObjectFromMessage(CodedInputStream cis) throws
10431069
if (builder.hasCacheBlocks()) {
10441070
this.cacheBlocks = builder.getCacheBlocks();
10451071
}
1072+
if (builder.hasIncludeStartRow()) {
1073+
this.includeStartRow = builder.getIncludeStartRow();
1074+
}
1075+
if (builder.hasIncludeStopRow()) {
1076+
this.includeStopRow = builder.getIncludeStopRow();
1077+
}
10461078
return this;
10471079
}
10481080

‎hbase-rest/src/main/protobuf/ScannerMessage.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ message Scanner {
3131
repeated string labels = 10;
3232
optional bool cacheBlocks = 11; // server side block caching hint
3333
optional int32 limit = 12;
34+
optional bool includeStartRow = 13;
35+
optional bool includeStopRow = 14;
3436
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,4 +399,155 @@ public void deleteNonExistent() throws IOException {
399399
Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
400400
assertEquals(404, response.getCode());
401401
}
402+
403+
@Test
404+
public void testScannerWithIncludeStartStopRowXML() throws IOException, JAXBException {
405+
final int BATCH_SIZE = 5;
406+
// new scanner
407+
ScannerModel model = new ScannerModel();
408+
// model.setBatch(BATCH_SIZE);
409+
model.addColumn(Bytes.toBytes(COLUMN_1));
410+
model.setStartRow(Bytes.toBytes("aaa"));
411+
model.setEndRow(Bytes.toBytes("aae"));
412+
StringWriter writer = new StringWriter();
413+
marshaller.marshal(model, writer);
414+
byte[] body = Bytes.toBytes(writer.toString());
415+
416+
conf.set("hbase.rest.readonly", "false");
417+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
418+
assertEquals(201, response.getCode());
419+
String scannerURI = response.getLocation();
420+
assertNotNull(scannerURI);
421+
422+
// get a cell set
423+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
424+
assertEquals(200, response.getCode());
425+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
426+
CellSetModel cellSet =
427+
(CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
428+
429+
assertEquals(4, countCellSet(cellSet));
430+
431+
// test with include the start row false
432+
model.setIncludeStartRow(false);
433+
writer = new StringWriter();
434+
marshaller.marshal(model, writer);
435+
body = Bytes.toBytes(writer.toString());
436+
437+
conf.set("hbase.rest.readonly", "false");
438+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
439+
assertEquals(201, response.getCode());
440+
scannerURI = response.getLocation();
441+
assertNotNull(scannerURI);
442+
443+
// get a cell set
444+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
445+
assertEquals(200, response.getCode());
446+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
447+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
448+
449+
assertEquals(3, countCellSet(cellSet));
450+
451+
// test with include stop row true and start row false
452+
model.setIncludeStartRow(false);
453+
model.setIncludeStopRow(true);
454+
writer = new StringWriter();
455+
marshaller.marshal(model, writer);
456+
body = Bytes.toBytes(writer.toString());
457+
458+
conf.set("hbase.rest.readonly", "false");
459+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
460+
assertEquals(201, response.getCode());
461+
scannerURI = response.getLocation();
462+
assertNotNull(scannerURI);
463+
464+
// get a cell set
465+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
466+
assertEquals(200, response.getCode());
467+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
468+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
469+
470+
assertEquals(4, countCellSet(cellSet));
471+
472+
// test with including the start row true and stop row true
473+
model.setIncludeStartRow(true);
474+
model.setIncludeStopRow(true);
475+
writer = new StringWriter();
476+
marshaller.marshal(model, writer);
477+
body = Bytes.toBytes(writer.toString());
478+
479+
conf.set("hbase.rest.readonly", "false");
480+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
481+
assertEquals(201, response.getCode());
482+
scannerURI = response.getLocation();
483+
assertNotNull(scannerURI);
484+
485+
// get a cell set
486+
response = client.get(scannerURI, Constants.MIMETYPE_XML);
487+
assertEquals(200, response.getCode());
488+
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
489+
cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
490+
491+
assertEquals(5, countCellSet(cellSet));
492+
}
493+
494+
@Test
495+
public void testScannerWithIncludeStartStopRowPB() throws IOException {
496+
final int BATCH_SIZE = 10;
497+
// new scanner
498+
ScannerModel model = new ScannerModel();
499+
// model.setBatch(BATCH_SIZE);
500+
model.addColumn(Bytes.toBytes(COLUMN_1));
501+
model.setStartRow(Bytes.toBytes("aaa"));
502+
model.setEndRow(Bytes.toBytes("aae"));
503+
504+
// test put operation is forbidden in read-only mode
505+
conf.set("hbase.rest.readonly", "false");
506+
Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
507+
model.createProtobufOutput());
508+
assertEquals(201, response.getCode());
509+
String scannerURI = response.getLocation();
510+
assertNotNull(scannerURI);
511+
512+
// get a cell set
513+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
514+
assertEquals(200, response.getCode());
515+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
516+
CellSetModel cellSet = new CellSetModel();
517+
cellSet.getObjectFromMessage(response.getBody());
518+
assertEquals(4, countCellSet(cellSet));
519+
520+
// test with include start row false
521+
model.setIncludeStartRow(false);
522+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
523+
model.createProtobufOutput());
524+
assertEquals(201, response.getCode());
525+
scannerURI = response.getLocation();
526+
assertNotNull(scannerURI);
527+
528+
// get a cell set
529+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
530+
assertEquals(200, response.getCode());
531+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
532+
cellSet = new CellSetModel();
533+
cellSet.getObjectFromMessage(response.getBody());
534+
assertEquals(3, countCellSet(cellSet));
535+
536+
// test with include stop row true
537+
model.setIncludeStartRow(true);
538+
model.setIncludeStopRow(true);
539+
response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
540+
model.createProtobufOutput());
541+
assertEquals(201, response.getCode());
542+
scannerURI = response.getLocation();
543+
assertNotNull(scannerURI);
544+
545+
// get a cell set
546+
response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
547+
assertEquals(200, response.getCode());
548+
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
549+
cellSet = new CellSetModel();
550+
cellSet.getObjectFromMessage(response.getBody());
551+
assertEquals(5, countCellSet(cellSet));
552+
}
402553
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,4 +670,82 @@ public void testLongLivedScan() throws Exception {
670670
Thread.sleep(trialPause);
671671
}
672672
}
673+
674+
@Test
675+
public void testScanWithInlcudeStartStopRow() throws Exception {
676+
int numTrials = 6;
677+
678+
// Truncate the test table for inserting test scenarios rows keys
679+
TEST_UTIL.getAdmin().disableTable(TABLE);
680+
TEST_UTIL.getAdmin().truncateTable(TABLE, false);
681+
String row = "testrow";
682+
683+
try (Table table = TEST_UTIL.getConnection().getTable(TABLE)) {
684+
List<Put> puts = new ArrayList<>();
685+
Put put = null;
686+
for (int i = 1; i <= numTrials; i++) {
687+
put = new Put(Bytes.toBytes(row + i));
688+
put.addColumn(COLUMN_1, QUALIFIER_1, TS_2, Bytes.toBytes("testvalue" + i));
689+
puts.add(put);
690+
}
691+
table.put(puts);
692+
}
693+
694+
remoteTable =
695+
new RemoteHTable(new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())),
696+
TEST_UTIL.getConfiguration(), TABLE.toBytes());
697+
698+
Scan scan =
699+
new Scan().withStartRow(Bytes.toBytes(row + "1")).withStopRow(Bytes.toBytes(row + "5"));
700+
701+
ResultScanner scanner = remoteTable.getScanner(scan);
702+
Iterator<Result> resultIterator = scanner.iterator();
703+
int counter = 0;
704+
while (resultIterator.hasNext()) {
705+
byte[] row1 = resultIterator.next().getRow();
706+
System.out.println(Bytes.toString(row1));
707+
counter++;
708+
}
709+
assertEquals(4, counter);
710+
711+
// test with include start row false
712+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
713+
.withStopRow(Bytes.toBytes(row + "5"));
714+
scanner = remoteTable.getScanner(scan);
715+
resultIterator = scanner.iterator();
716+
counter = 0;
717+
while (resultIterator.hasNext()) {
718+
byte[] row1 = resultIterator.next().getRow();
719+
System.out.println(Bytes.toString(row1));
720+
counter++;
721+
}
722+
assertEquals(3, counter);
723+
724+
// test with include start row false and stop row true
725+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
726+
.withStopRow(Bytes.toBytes(row + "5"), true);
727+
scanner = remoteTable.getScanner(scan);
728+
resultIterator = scanner.iterator();
729+
counter = 0;
730+
while (resultIterator.hasNext()) {
731+
byte[] row1 = resultIterator.next().getRow();
732+
System.out.println(Bytes.toString(row1));
733+
counter++;
734+
}
735+
assertEquals(4, counter);
736+
737+
// test with include start row true and stop row true
738+
scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), true)
739+
.withStopRow(Bytes.toBytes(row + "5"), true);
740+
scanner = remoteTable.getScanner(scan);
741+
resultIterator = scanner.iterator();
742+
counter = 0;
743+
while (resultIterator.hasNext()) {
744+
byte[] row1 = resultIterator.next().getRow();
745+
System.out.println(Bytes.toString(row1));
746+
counter++;
747+
}
748+
assertEquals(5, counter);
749+
}
750+
673751
}

‎hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ public TestScannerModel() throws Exception {
6262
AS_JSON = "{\"batch\":100,\"caching\":1000,\"cacheBlocks\":false,\"endRow\":\"enp5eng=\","
6363
+ "\"endTime\":1245393318192,\"maxVersions\":2147483647,\"startRow\":\"YWJyYWNhZGFicmE=\","
6464
+ "\"startTime\":1245219839331,\"column\":[\"Y29sdW1uMQ==\",\"Y29sdW1uMjpmb28=\"],"
65-
+ "\"labels\":[\"private\",\"public\"]," + "\"limit\":10000}";
65+
+ "\"labels\":[\"private\",\"public\"]," + "\"limit\":10000,"
66+
+ "\"includeStartRow\":true,\"includeStopRow\":false}";
6667

6768
AS_PB = "CgthYnJhY2FkYWJyYRIFenp5engaB2NvbHVtbjEaC2NvbHVtbjI6Zm9vIGQo47qL554kMLDi57mfJDj"
6869
+ "/////B0joB1IHcHJpdmF0ZVIGcHVibGljWABgkE4=";

0 commit comments

Comments
 (0)