Skip to content

Upgrade to lucene-5.3.0. #13239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 1, 2015
Merged

Upgrade to lucene-5.3.0. #13239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;

import java.io.IOException;

Expand Down Expand Up @@ -60,7 +59,7 @@ public boolean equals(Object obj) {
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
return new ConstantScoreWeight(this) {
@Override
public Scorer scorer(LeafReaderContext context, final Bits acceptDocs) throws IOException {
public Scorer scorer(LeafReaderContext context) throws IOException {
final int maxDoc = context.reader().maxDoc();
if (context.docBase + maxDoc <= minDoc) {
return null;
Expand Down Expand Up @@ -89,12 +88,6 @@ public int advance(int target) throws IOException {
} else {
doc = target;
}
while (doc < maxDoc) {
if (acceptDocs == null || acceptDocs.get(doc)) {
break;
}
doc += 1;
}
if (doc >= maxDoc) {
doc = NO_MORE_DOCS;
}
Expand Down
33 changes: 17 additions & 16 deletions core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ protected Query getFieldQuery(String field, String queryText, int slop) throws P
if (q != null) {
added = true;
applyBoost(mField, q);
applySlop(q, slop);
q = applySlop(q, slop);
disMaxQuery.add(q);
}
}
Expand All @@ -293,7 +293,7 @@ protected Query getFieldQuery(String field, String queryText, int slop) throws P
Query q = super.getFieldQuery(mField, queryText, slop);
if (q != null) {
applyBoost(mField, q);
applySlop(q, slop);
q = applySlop(q, slop);
clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD));
}
}
Expand Down Expand Up @@ -718,15 +718,6 @@ private Query getPossiblyAnalyzedWildcardQuery(String field, String termStr) thr
return super.getWildcardQuery(field, aggStr.toString());
}

@Override
protected WildcardQuery newWildcardQuery(Term t) {
// Backport: https://issues.apache.org/jira/browse/LUCENE-6677
assert Version.LATEST == Version.LUCENE_5_2_1;
WildcardQuery query = new WildcardQuery(t, maxDeterminizedStates);
query.setRewriteMethod(multiTermRewriteMethod);
return query;
}

@Override
protected Query getRegexpQuery(String field, String termStr) throws ParseException {
if (lowercaseExpandedTerms) {
Expand Down Expand Up @@ -815,14 +806,24 @@ private void applyBoost(String field, Query q) {
}
}

private void applySlop(Query q, int slop) {
if (q instanceof FilteredQuery) {
applySlop(((FilteredQuery)q).getQuery(), slop);
}
private Query applySlop(Query q, int slop) {
if (q instanceof PhraseQuery) {
((PhraseQuery) q).setSlop(slop);
PhraseQuery pq = (PhraseQuery) q;
PhraseQuery.Builder builder = new PhraseQuery.Builder();
builder.setSlop(slop);
final Term[] terms = pq.getTerms();
final int[] positions = pq.getPositions();
for (int i = 0; i < terms.length; ++i) {
builder.add(terms[i], positions[i]);
}
pq = builder.build();
pq.setBoost(q.getBoost());
return pq;
} else if (q instanceof MultiPhraseQuery) {
((MultiPhraseQuery) q).setSlop(slop);
return q;
} else {
return q;
}
}

Expand Down
153 changes: 0 additions & 153 deletions ...rc/main/java/org/apache/lucene/search/postingshighlight/CustomSeparatorBreakIterator.java
View file
Open in desktop

This file was deleted.

62 changes: 33 additions & 29 deletions core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.lucene.store.*;
import org.apache.lucene.util.*;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.LimitedFiniteStringsIterator;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.fst.*;
Expand Down Expand Up @@ -465,16 +466,12 @@ public void build(InputIterator iterator) throws IOException {
byte buffer[] = new byte[8];
try {
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
BytesRef surfaceForm;

while ((surfaceForm = iterator.next()) != null) {
Set<IntsRef> paths = toFiniteStrings(surfaceForm, ts2a);

maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, paths.size());

for (IntsRef path : paths) {

Util.toBytesRef(path, scratch);
for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) {
LimitedFiniteStringsIterator finiteStrings =
new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
for (IntsRef string; (string = finiteStrings.next()) != null; count++) {
Util.toBytesRef(string, scratch);

// length of the analyzed text (FST input)
if (scratch.length() > Short.MAX_VALUE-2) {
Expand Down Expand Up @@ -526,7 +523,7 @@ public void build(InputIterator iterator) throws IOException {

writer.write(buffer, 0, output.getPosition());
}
count++;
maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, finiteStrings.size());
}
writer.close();

Expand Down Expand Up @@ -912,23 +909,17 @@ protected List<FSTUtil.Path<Pair<Long,BytesRef>>> getFullPrefixPaths(List<FSTUti
return prefixPaths;
}

public final Set<IntsRef> toFiniteStrings(final BytesRef surfaceForm, final TokenStreamToAutomaton ts2a) throws IOException {
// Analyze surface form:
TokenStream ts = indexAnalyzer.tokenStream("", surfaceForm.utf8ToString());
return toFiniteStrings(ts2a, ts);
final Automaton toAutomaton(final BytesRef surfaceForm, final TokenStreamToAutomaton ts2a) throws IOException {
try (TokenStream ts = indexAnalyzer.tokenStream("", surfaceForm.utf8ToString())) {
return toAutomaton(ts, ts2a);
}
}

public final Set<IntsRef> toFiniteStrings(final TokenStreamToAutomaton ts2a, final TokenStream ts) throws IOException {
Automaton automaton = null;
try {

// Create corresponding automaton: labels are bytes
// from each analyzed token, with byte 0 used as
// separator between tokens:
automaton = ts2a.toAutomaton(ts);
} finally {
IOUtils.closeWhileHandlingException(ts);
}
final Automaton toAutomaton(TokenStream ts, final TokenStreamToAutomaton ts2a) throws IOException {
// Create corresponding automaton: labels are bytes
// from each analyzed token, with byte 0 used as
// separator between tokens:
Automaton automaton = ts2a.toAutomaton(ts);

automaton = replaceSep(automaton);
automaton = convertAutomaton(automaton);
Expand All @@ -940,11 +931,24 @@ public final Set<IntsRef> toFiniteStrings(final TokenStreamToAutomaton ts2a, fin
// more than one path, eg if the analyzer created a
// graph using SynFilter or WDF):

// TODO: we could walk & add simultaneously, so we
// don't have to alloc [possibly biggish]
// intermediate HashSet in RAM:
return automaton;
}

return Operations.getFiniteStrings(automaton, maxGraphExpansions);
// EDIT: Adrien, needed by lookup providers
// NOTE: these XForks are unmaintainable, we need to get rid of them...
public Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException {
final TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();
Automaton automaton;
try (TokenStream ts = stream) {
automaton = toAutomaton(ts, ts2a);
}
LimitedFiniteStringsIterator finiteStrings =
new LimitedFiniteStringsIterator(automaton, maxGraphExpansions);
Set<IntsRef> set = new HashSet<>();
for (IntsRef string = finiteStrings.next(); string != null; string = finiteStrings.next()) {
set.add(IntsRef.deepCopyOf(string));
}
return Collections.unmodifiableSet(set);
}

final Automaton toLookupAutomaton(final CharSequence key) throws IOException {
Expand Down
Loading