Skip to content

Enable convert processor to support Long and Double. #27957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 3, 2018
Merged

Enable convert processor to support Long and Double. #27957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions docs/reference/ingest/ingest-node.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,16 +778,17 @@ Accepts a single value or an array of values.
Converts an existing field's value to a different type, such as converting a string to an integer.
If the field value is an array, all members will be converted.

The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`.
The supported types include: `integer`, `long`, `float`, `double`, `string`, `boolean`, and `auto`.

Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.

Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And
a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot
be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In
such a case, `target_field` will still be updated with the unconverted field value.
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. Do note
that float takes precedence of double in `auto`. A value of `"242.15"` will "automatically" be converted to
`242.15` of type `float`. If a provided field cannot be appropriately converted, the Convert Processor will
still process successfully and leave the field value as-is. In such a case, `target_field` will
still be updated with the unconverted field value.

[[convert-options]]
.Convert Options
Expand Down
26 changes: 25 additions & 1 deletion modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,24 @@ public Object convert(Object value) {
}

}
}, LONG {
@Override
public Object convert(Object value) {
try {
return Long.parseLong(value.toString());
} catch(NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to long", e);
}
}
}, DOUBLE {
@Override
public Object convert(Object value) {
try {
return Double.parseDouble(value.toString());
} catch(NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to double", e);
}
}
}, FLOAT {
@Override
public Object convert(Object value) {
Expand Down Expand Up @@ -81,13 +99,19 @@ public Object convert(Object value) {
}
try {
return BOOLEAN.convert(value);
} catch (IllegalArgumentException e) { }
} catch (IllegalArgumentException e) {}
try {
return INTEGER.convert(value);
} catch (IllegalArgumentException e) {}
try {
return LONG.convert(value);
} catch (IllegalArgumentException e) {}
try {
return FLOAT.convert(value);
} catch (IllegalArgumentException e) {}
try {
return DOUBLE.convert(value);
} catch (IllegalArgumentException e) {}
return value;
}
};
Expand Down
119 changes: 119 additions & 0 deletions ...es/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance;
import static org.hamcrest.Matchers.not;

public class ConvertProcessorTests extends ESTestCase {

Expand Down Expand Up @@ -79,6 +80,92 @@ public void testConvertIntError() throws Exception {
}
}

public void testConvertLong() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Long> expectedResult = new HashMap<>();
long randomLong = randomLong();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomLong);
expectedResult.put(fieldName, randomLong);

Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong));
}

public void testConvertLongList() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int numItems = randomIntBetween(1, 10);
List<String> fieldValue = new ArrayList<>();
List<Long> expectedList = new ArrayList<>();
for (int j = 0; j < numItems; j++) {
long randomLong = randomLong();
fieldValue.add(Long.toString(randomLong));
expectedList.add(randomLong);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}

public void testConvertLongError() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);

Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long"));
}
}

public void testConvertDouble() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Double> expectedResult = new HashMap<>();
double randomDouble = randomDouble();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomDouble);
expectedResult.put(fieldName, randomDouble);

Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Double.class), equalTo(randomDouble));
}

public void testConvertDoubleList() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int numItems = randomIntBetween(1, 10);
List<String> fieldValue = new ArrayList<>();
List<Double> expectedList = new ArrayList<>();
for (int j = 0; j < numItems; j++) {
double randomDouble = randomDouble();
fieldValue.add(Double.toString(randomDouble));
expectedList.add(randomDouble);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}

public void testConvertDoubleError() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);

Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to double"));
}
}

public void testConvertFloat() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Float> expectedResult = new HashMap<>();
Expand Down Expand Up @@ -231,6 +318,16 @@ public void testConvertStringList() throws Exception {
randomValue = randomBoolean;
randomValueString = Boolean.toString(randomBoolean);
break;
case 3:
long randomLong = randomLong();
randomValue = randomLong;
randomValueString = Long.toString(randomLong);
break;
case 4:
double randomDouble = randomDouble();
randomValue = randomDouble;
randomValueString = Double.toString(randomDouble);
break;
default:
throw new UnsupportedOperationException();
}
Expand Down Expand Up @@ -342,6 +439,28 @@ public void testAutoConvertMatchInteger() throws Exception {
assertThat(convertedValue, equalTo(randomInt));
}

public void testAutoConvertMatchLong() throws Exception {
long randomLong = randomLong();
String randomString = Long.toString(randomLong);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomLong));
}

public void testAutoConvertDoubleNotMatched() throws Exception {
double randomDouble = randomDouble();
String randomString = Double.toString(randomDouble);
float randomFloat = Float.parseFloat(randomString);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, not(randomDouble));
assertThat(convertedValue, equalTo(randomFloat));
}

public void testAutoConvertMatchFloat() throws Exception {
float randomFloat = randomFloat();
String randomString = Float.toString(randomFloat);
Expand Down