Skip to content

Commit eeac6d2

Browse files
shaiejimczi
shaie
authored and
jimczi
committed
Add BreakIteratorBoundaryScanner support for FVH (#23248)
This commit adds a boundary_scanner property to the search highlight request so the user can specify different boundary scanners: * `chars` (default, current behavior) * `word` Use a WordBreakIterator * `sentence` Use a SentenceBreakIterator This commit also adds "boundary_scanner_locale" to define which locale should be used when scanning the text.
1 parent 25a9a7e commit eeac6d2

File tree

7 files changed

+300
-21
lines changed

7 files changed

+300
-21
lines changed

‎core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.lucene.search.highlight.SimpleFragmenter;
2323
import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
24+
import org.elasticsearch.Version;
2425
import org.elasticsearch.action.support.ToXContentToBytes;
2526
import org.elasticsearch.common.ParseField;
2627
import org.elasticsearch.common.ParsingException;
@@ -32,10 +33,12 @@
3233
import org.elasticsearch.common.xcontent.XContentParser;
3334
import org.elasticsearch.index.query.QueryBuilder;
3435
import org.elasticsearch.index.query.QueryParseContext;
36+
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.BoundaryScannerType;
3537
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Order;
3638

3739
import java.io.IOException;
3840
import java.util.Arrays;
41+
import java.util.Locale;
3942
import java.util.Map;
4043
import java.util.Objects;
4144
import java.util.function.BiFunction;
@@ -57,8 +60,10 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
5760
public static final ParseField NUMBER_OF_FRAGMENTS_FIELD = new ParseField("number_of_fragments");
5861
public static final ParseField ENCODER_FIELD = new ParseField("encoder");
5962
public static final ParseField REQUIRE_FIELD_MATCH_FIELD = new ParseField("require_field_match");
63+
public static final ParseField BOUNDARY_SCANNER_FIELD = new ParseField("boundary_scanner");
6064
public static final ParseField BOUNDARY_MAX_SCAN_FIELD = new ParseField("boundary_max_scan");
6165
public static final ParseField BOUNDARY_CHARS_FIELD = new ParseField("boundary_chars");
66+
public static final ParseField BOUNDARY_SCANNER_LOCALE_FIELD = new ParseField("boundary_scanner_locale");
6267
public static final ParseField TYPE_FIELD = new ParseField("type");
6368
public static final ParseField FRAGMENTER_FIELD = new ParseField("fragmenter");
6469
public static final ParseField NO_MATCH_SIZE_FIELD = new ParseField("no_match_size");
@@ -88,10 +93,14 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
8893

8994
protected Boolean forceSource;
9095

96+
protected BoundaryScannerType boundaryScannerType;
97+
9198
protected Integer boundaryMaxScan;
9299

93100
protected char[] boundaryChars;
94101

102+
protected Locale boundaryScannerLocale;
103+
95104
protected Integer noMatchSize;
96105

97106
protected Integer phraseLimit;
@@ -119,10 +128,18 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException {
119128
order(in.readOptionalWriteable(Order::readFromStream));
120129
highlightFilter(in.readOptionalBoolean());
121130
forceSource(in.readOptionalBoolean());
131+
if (in.getVersion().onOrAfter(Version.V_5_4_0_UNRELEASED)) {
132+
boundaryScannerType(in.readOptionalWriteable(BoundaryScannerType::readFromStream));
133+
}
122134
boundaryMaxScan(in.readOptionalVInt());
123135
if (in.readBoolean()) {
124136
boundaryChars(in.readString().toCharArray());
125137
}
138+
if (in.getVersion().onOrAfter(Version.V_5_4_0_UNRELEASED)) {
139+
if (in.readBoolean()) {
140+
boundaryScannerLocale(in.readString());
141+
}
142+
}
126143
noMatchSize(in.readOptionalVInt());
127144
phraseLimit(in.readOptionalVInt());
128145
if (in.readBoolean()) {
@@ -150,12 +167,22 @@ public final void writeTo(StreamOutput out) throws IOException {
150167
out.writeOptionalWriteable(order);
151168
out.writeOptionalBoolean(highlightFilter);
152169
out.writeOptionalBoolean(forceSource);
170+
if (out.getVersion().onOrAfter(Version.V_5_4_0_UNRELEASED)) {
171+
out.writeOptionalWriteable(boundaryScannerType);
172+
}
153173
out.writeOptionalVInt(boundaryMaxScan);
154174
boolean hasBounaryChars = boundaryChars != null;
155175
out.writeBoolean(hasBounaryChars);
156176
if (hasBounaryChars) {
157177
out.writeString(String.valueOf(boundaryChars));
158178
}
179+
if (out.getVersion().onOrAfter(Version.V_5_4_0_UNRELEASED)) {
180+
boolean hasBoundaryScannerLocale = boundaryScannerLocale != null;
181+
out.writeBoolean(hasBoundaryScannerLocale);
182+
if (hasBoundaryScannerLocale) {
183+
out.writeString(boundaryScannerLocale.toLanguageTag());
184+
}
185+
}
159186
out.writeOptionalVInt(noMatchSize);
160187
out.writeOptionalVInt(phraseLimit);
161188
boolean hasOptions = options != null;
@@ -331,6 +358,33 @@ public Boolean highlightFilter() {
331358
return this.highlightFilter;
332359
}
333360

361+
/**
362+
* When using the highlighterType <tt>fvh</tt> this setting
363+
* controls which scanner to use for fragment boundaries, and defaults to "simple".
364+
*/
365+
@SuppressWarnings("unchecked")
366+
public HB boundaryScannerType(String boundaryScannerType) {
367+
this.boundaryScannerType = BoundaryScannerType.fromString(boundaryScannerType);
368+
return (HB) this;
369+
}
370+
371+
/**
372+
* When using the highlighterType <tt>fvh</tt> this setting
373+
* controls which scanner to use for fragment boundaries, and defaults to "simple".
374+
*/
375+
@SuppressWarnings("unchecked")
376+
public HB boundaryScannerType(BoundaryScannerType boundaryScannerType) {
377+
this.boundaryScannerType = boundaryScannerType;
378+
return (HB) this;
379+
}
380+
381+
/**
382+
* @return the value set by {@link #boundaryScannerType(String)}
383+
*/
384+
public BoundaryScannerType boundaryScannerType() {
385+
return this.boundaryScannerType;
386+
}
387+
334388
/**
335389
* When using the highlighterType <tt>fvh</tt> this setting
336390
* controls how far to look for boundary characters, and defaults to 20.
@@ -366,6 +420,25 @@ public char[] boundaryChars() {
366420
return this.boundaryChars;
367421
}
368422

423+
/**
424+
* When using the highlighterType <tt>fvh</tt> and boundaryScannerType <tt>break_iterator</tt>, this setting
425+
* controls the locale to use by the BreakIterator, defaults to "root".
426+
*/
427+
@SuppressWarnings("unchecked")
428+
public HB boundaryScannerLocale(String boundaryScannerLocale) {
429+
if (boundaryScannerLocale != null) {
430+
this.boundaryScannerLocale = Locale.forLanguageTag(boundaryScannerLocale);
431+
}
432+
return (HB) this;
433+
}
434+
435+
/**
436+
* @return the value set by {@link #boundaryScannerLocale(String)}
437+
*/
438+
public Locale boundaryScannerLocale() {
439+
return this.boundaryScannerLocale;
440+
}
441+
369442
/**
370443
* Allows to set custom options for custom highlighters.
371444
*/
@@ -491,12 +564,18 @@ void commonOptionsToXContent(XContentBuilder builder) throws IOException {
491564
if (highlightFilter != null) {
492565
builder.field(HIGHLIGHT_FILTER_FIELD.getPreferredName(), highlightFilter);
493566
}
567+
if (boundaryScannerType != null) {
568+
builder.field(BOUNDARY_SCANNER_FIELD.getPreferredName(), boundaryScannerType.name());
569+
}
494570
if (boundaryMaxScan != null) {
495571
builder.field(BOUNDARY_MAX_SCAN_FIELD.getPreferredName(), boundaryMaxScan);
496572
}
497573
if (boundaryChars != null) {
498574
builder.field(BOUNDARY_CHARS_FIELD.getPreferredName(), new String(boundaryChars));
499575
}
576+
if (boundaryScannerLocale != null) {
577+
builder.field(BOUNDARY_SCANNER_LOCALE_FIELD.getPreferredName(), boundaryScannerLocale.toLanguageTag());
578+
}
500579
if (options != null && options.size() > 0) {
501580
builder.field(OPTIONS_FIELD.getPreferredName(), options);
502581
}
@@ -523,8 +602,10 @@ static <HB extends AbstractHighlighterBuilder<HB>> BiFunction<QueryParseContext,
523602
parser.declareInt(HB::fragmentSize, FRAGMENT_SIZE_FIELD);
524603
parser.declareInt(HB::numOfFragments, NUMBER_OF_FRAGMENTS_FIELD);
525604
parser.declareBoolean(HB::requireFieldMatch, REQUIRE_FIELD_MATCH_FIELD);
605+
parser.declareString(HB::boundaryScannerType, BOUNDARY_SCANNER_FIELD);
526606
parser.declareInt(HB::boundaryMaxScan, BOUNDARY_MAX_SCAN_FIELD);
527607
parser.declareString((HB hb, String bc) -> hb.boundaryChars(bc.toCharArray()) , BOUNDARY_CHARS_FIELD);
608+
parser.declareString(HB::boundaryScannerLocale, BOUNDARY_SCANNER_LOCALE_FIELD);
528609
parser.declareString(HB::highlighterType, TYPE_FIELD);
529610
parser.declareString(HB::fragmenter, FRAGMENTER_FIELD);
530611
parser.declareInt(HB::noMatchSize, NO_MATCH_SIZE_FIELD);
@@ -562,8 +643,8 @@ static <HB extends AbstractHighlighterBuilder<HB>> BiFunction<QueryParseContext,
562643
public final int hashCode() {
563644
return Objects.hash(getClass(), Arrays.hashCode(preTags), Arrays.hashCode(postTags), fragmentSize,
564645
numOfFragments, highlighterType, fragmenter, highlightQuery, order, highlightFilter,
565-
forceSource, boundaryMaxScan, Arrays.hashCode(boundaryChars), noMatchSize,
566-
phraseLimit, options, requireFieldMatch, doHashCode());
646+
forceSource, boundaryScannerType, boundaryMaxScan, Arrays.hashCode(boundaryChars), boundaryScannerLocale,
647+
noMatchSize, phraseLimit, options, requireFieldMatch, doHashCode());
567648
}
568649

569650
/**
@@ -591,8 +672,10 @@ public final boolean equals(Object obj) {
591672
Objects.equals(order, other.order) &&
592673
Objects.equals(highlightFilter, other.highlightFilter) &&
593674
Objects.equals(forceSource, other.forceSource) &&
675+
Objects.equals(boundaryScannerType, other.boundaryScannerType) &&
594676
Objects.equals(boundaryMaxScan, other.boundaryMaxScan) &&
595677
Arrays.equals(boundaryChars, other.boundaryChars) &&
678+
Objects.equals(boundaryScannerLocale, other.boundaryScannerLocale) &&
596679
Objects.equals(noMatchSize, other.noMatchSize) &&
597680
Objects.equals(phraseLimit, other.phraseLimit) &&
598681
Objects.equals(options, other.options) &&

‎core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.lucene.search.highlight.Encoder;
2222
import org.apache.lucene.search.vectorhighlight.BaseFragmentsBuilder;
2323
import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
24+
import org.apache.lucene.search.vectorhighlight.BreakIteratorBoundaryScanner;
2425
import org.apache.lucene.search.vectorhighlight.CustomFieldQuery;
2526
import org.apache.lucene.search.vectorhighlight.FieldFragList;
2627
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
@@ -38,15 +39,23 @@
3839
import org.elasticsearch.index.mapper.FieldMapper;
3940
import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
4041
import org.elasticsearch.search.fetch.FetchSubPhase;
42+
import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.Field;
43+
import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight.FieldOptions;
4144
import org.elasticsearch.search.internal.SearchContext;
4245

46+
import java.text.BreakIterator;
4347
import java.util.Collections;
4448
import java.util.HashMap;
49+
import java.util.Locale;
4550
import java.util.Map;
4651

4752
public class FastVectorHighlighter implements Highlighter {
4853

49-
private static final SimpleBoundaryScanner DEFAULT_BOUNDARY_SCANNER = new SimpleBoundaryScanner();
54+
private static final BoundaryScanner DEFAULT_SIMPLE_BOUNDARY_SCANNER = new SimpleBoundaryScanner();
55+
private static final BoundaryScanner DEFAULT_SENTENCE_BOUNDARY_SCANNER = new BreakIteratorBoundaryScanner(
56+
BreakIterator.getSentenceInstance(Locale.ROOT));
57+
private static final BoundaryScanner DEFAULT_WORD_BOUNDARY_SCANNER = new BreakIteratorBoundaryScanner(
58+
BreakIterator.getWordInstance(Locale.ROOT));
5059

5160
public static final Setting<Boolean> SETTING_TV_HIGHLIGHT_MULTI_VALUE = Setting.boolSetting("search.highlight.term_vector_multi_value",
5261
true, Setting.Property.NodeScope);
@@ -105,12 +114,7 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
105114
FragListBuilder fragListBuilder;
106115
BaseFragmentsBuilder fragmentsBuilder;
107116

108-
BoundaryScanner boundaryScanner = DEFAULT_BOUNDARY_SCANNER;
109-
if (field.fieldOptions().boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
110-
|| field.fieldOptions().boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
111-
boundaryScanner = new SimpleBoundaryScanner(field.fieldOptions().boundaryMaxScan(),
112-
field.fieldOptions().boundaryChars());
113-
}
117+
final BoundaryScanner boundaryScanner = getBoundaryScanner(field);
114118
boolean forceSource = context.highlight().forceSource(field);
115119
if (field.fieldOptions().numberOfFragments() == 0) {
116120
fragListBuilder = new SingleFragListBuilder();
@@ -206,6 +210,29 @@ public boolean canHighlight(FieldMapper fieldMapper) {
206210
&& fieldMapper.fieldType().storeTermVectorPositions();
207211
}
208212

213+
private static BoundaryScanner getBoundaryScanner(Field field) {
214+
final FieldOptions fieldOptions = field.fieldOptions();
215+
final Locale boundaryScannerLocale = fieldOptions.boundaryScannerLocale();
216+
switch(fieldOptions.boundaryScannerType()) {
217+
case SENTENCE:
218+
if (boundaryScannerLocale != null) {
219+
return new BreakIteratorBoundaryScanner(BreakIterator.getSentenceInstance(boundaryScannerLocale));
220+
}
221+
return DEFAULT_SENTENCE_BOUNDARY_SCANNER;
222+
case WORD:
223+
if (boundaryScannerLocale != null) {
224+
return new BreakIteratorBoundaryScanner(BreakIterator.getWordInstance(boundaryScannerLocale));
225+
}
226+
return DEFAULT_WORD_BOUNDARY_SCANNER;
227+
default:
228+
if (fieldOptions.boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
229+
|| fieldOptions.boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
230+
return new SimpleBoundaryScanner(fieldOptions.boundaryMaxScan(), fieldOptions.boundaryChars());
231+
}
232+
return DEFAULT_SIMPLE_BOUNDARY_SCANNER;
233+
}
234+
}
235+
209236
private class MapperHighlightEntry {
210237
public FragListBuilder fragListBuilder;
211238
public FragmentsBuilder fragmentsBuilder;

‎core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilder.java‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
9595
.preTags(DEFAULT_PRE_TAGS).postTags(DEFAULT_POST_TAGS).scoreOrdered(DEFAULT_SCORE_ORDERED)
9696
.highlightFilter(DEFAULT_HIGHLIGHT_FILTER).requireFieldMatch(DEFAULT_REQUIRE_FIELD_MATCH)
9797
.forceSource(DEFAULT_FORCE_SOURCE).fragmentCharSize(DEFAULT_FRAGMENT_CHAR_SIZE)
98-
.numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS).encoder(DEFAULT_ENCODER)
98+
.numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS).encoder(DEFAULT_ENCODER).boundaryScannerType(BoundaryScannerType.CHARS)
9999
.boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN).boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS)
100-
.noMatchSize(DEFAULT_NO_MATCH_SIZE).phraseLimit(DEFAULT_PHRASE_LIMIT).build();
100+
.boundaryScannerLocale(Locale.ROOT).noMatchSize(DEFAULT_NO_MATCH_SIZE).phraseLimit(DEFAULT_PHRASE_LIMIT).build();
101101

102102
private final List<Field> fields = new ArrayList<>();
103103

@@ -327,12 +327,18 @@ private static void transferOptions(AbstractHighlighterBuilder highlighterBuilde
327327
if (highlighterBuilder.requireFieldMatch != null) {
328328
targetOptionsBuilder.requireFieldMatch(highlighterBuilder.requireFieldMatch);
329329
}
330+
if (highlighterBuilder.boundaryScannerType != null) {
331+
targetOptionsBuilder.boundaryScannerType(highlighterBuilder.boundaryScannerType);
332+
}
330333
if (highlighterBuilder.boundaryMaxScan != null) {
331334
targetOptionsBuilder.boundaryMaxScan(highlighterBuilder.boundaryMaxScan);
332335
}
333336
if (highlighterBuilder.boundaryChars != null) {
334337
targetOptionsBuilder.boundaryChars(convertCharArray(highlighterBuilder.boundaryChars));
335338
}
339+
if (highlighterBuilder.boundaryScannerLocale != null) {
340+
targetOptionsBuilder.boundaryScannerLocale(highlighterBuilder.boundaryScannerLocale);
341+
}
336342
if (highlighterBuilder.highlighterType != null) {
337343
targetOptionsBuilder.highlighterType(highlighterBuilder.highlighterType);
338344
}
@@ -522,4 +528,30 @@ public String toString() {
522528
return name().toLowerCase(Locale.ROOT);
523529
}
524530
}
531+
532+
public enum BoundaryScannerType implements Writeable {
533+
CHARS, WORD, SENTENCE;
534+
535+
public static BoundaryScannerType readFromStream(StreamInput in) throws IOException {
536+
int ordinal = in.readVInt();
537+
if (ordinal < 0 || ordinal >= values().length) {
538+
throw new IOException("Unknown BoundaryScannerType ordinal [" + ordinal + "]");
539+
}
540+
return values()[ordinal];
541+
}
542+
543+
@Override
544+
public void writeTo(StreamOutput out) throws IOException {
545+
out.writeVInt(this.ordinal());
546+
}
547+
548+
public static BoundaryScannerType fromString(String boundaryScannerType) {
549+
return valueOf(boundaryScannerType.toUpperCase(Locale.ROOT));
550+
}
551+
552+
@Override
553+
public String toString() {
554+
return name().toLowerCase(Locale.ROOT);
555+
}
556+
}
525557
}

0 commit comments

Comments
 (0)