From 867bc962ccd5ef6382cb374cc461394e47c7e0b2 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Thu, 14 Oct 2021 12:15:19 +0200 Subject: [PATCH 1/3] Add test cases for TripleSort. --- metafacture-triples/build.gradle | 7 + .../metafacture/triples/TripleSortTest.java | 256 ++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java diff --git a/metafacture-triples/build.gradle b/metafacture-triples/build.gradle index 821d2e18a..e8d5966c1 100644 --- a/metafacture-triples/build.gradle +++ b/metafacture-triples/build.gradle @@ -24,3 +24,10 @@ dependencies { testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.5.5' } + +test { + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} diff --git a/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java b/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java new file mode 100644 index 000000000..00b2b2c27 --- /dev/null +++ b/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java @@ -0,0 +1,256 @@ +/* + * Copyright 2021 hbz NRW + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.triples; + +import org.metafacture.framework.ObjectReceiver; +import org.metafacture.framework.objects.Triple; + +import org.junit.Rule; +import org.junit.Test; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.exceptions.base.MockitoAssertionError; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.Arrays; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public final class TripleSortTest { + + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private ObjectReceiver receiver; + + @Test + public void shouldSortByIncreasingSubject() { + assertSort( + t -> { + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s0 p1 o2", + "s0 p1 o1", + "s0 p2 o1", + "s1 p0 o2", + "s2 p1 o0" + ); + } + + @Test + public void shouldSortByDecreasingSubject() { + assertSort( + t -> { + t.setOrder(AbstractTripleSort.Order.DECREASING); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s2 p1 o0", + "s1 p0 o2", + "s0 p1 o2", + "s0 p1 o1", + "s0 p2 o1" + ); + } + + @Test + public void shouldSortByIncreasingPredicate() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.PREDICATE); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s1 p0 o2", + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s0 p2 o1" + ); + } + + @Test + public void shouldSortByDecreasingPredicate() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.PREDICATE); + t.setOrder(AbstractTripleSort.Order.DECREASING); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s0 p2 o1", + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2" + ); + } + + @Test + public void shouldSortByIncreasingObject() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.OBJECT); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s2 p1 o0", + "s0 p1 o1", + "s0 p2 o1", + "s0 p1 o2", + "s1 p0 o2" + ); + } + + @Test + public void shouldSortByDecreasingObject() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.OBJECT); + t.setOrder(AbstractTripleSort.Order.DECREASING); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s0 p1 o2", + "s1 p0 o2", + "s0 p1 o1", + "s0 p2 o1", + "s2 p1 o0" + ); + } + + @Test + public void shouldSortByIncreasingTriple() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.ALL); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s0 p1 o1", + "s0 p1 o2", + "s0 p2 o1", + "s1 p0 o2", + "s2 p1 o0" + ); + } + + @Test + public void shouldSortByDecreasingTriple() { + assertSort( + t -> { + t.setBy(AbstractTripleSort.Compare.ALL); + t.setOrder(AbstractTripleSort.Order.DECREASING); + }, + "s0 p1 o2", + "s2 p1 o0", + "s0 p1 o1", + "s1 p0 o2", + "s0 p2 o1", + // + "s2 p1 o0", + "s1 p0 o2", + "s0 p2 o1", + "s0 p1 o2", + "s0 p1 o1" + ); + } + + @Test + public void shouldNotSortNumerically() { + assertSort( + t -> { + }, + "10 p1 o2", + "2 p1 o0", + "0 p1 o1", + "101 p0 o2", + "11 p2 o1", + // + "0 p1 o1", + "10 p1 o2", + "101 p0 o2", + "11 p2 o1", + "2 p1 o0" + ); + } + + public void assertSort(final Consumer consumer, final String... triples) { + final BiConsumer> processor = (i, c) -> { + final int j = triples.length / 2; + + Arrays.stream(triples, i * j, (i + 1) * j).map(s -> { + final String[] t = s.split("\\s+"); + return new Triple(t[0], t[1], t[2]); + }).forEach(c); + }; + + final InOrder ordered = Mockito.inOrder(receiver); + + final TripleSort tripleSort = new TripleSort(); + tripleSort.setReceiver(receiver); + consumer.accept(tripleSort); + + processor.accept(0, tripleSort::process); + tripleSort.closeStream(); + + try { + processor.accept(1, t -> ordered.verify(receiver).process(t)); + ordered.verify(receiver).closeStream(); + + ordered.verifyNoMoreInteractions(); + Mockito.verifyNoMoreInteractions(receiver); + } + catch (final MockitoAssertionError e) { + System.out.println(Mockito.mockingDetails(receiver).printInvocations()); + throw e; + } + } + +} From 5a0d39300375be03dfda782fae58d76ffb21ea09 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Thu, 14 Oct 2021 15:06:26 +0200 Subject: [PATCH 2/3] Add option to sort triples numerically. Fixes #380. --- .../triples/AbstractTripleSort.java | 93 ++++++++----------- .../org/metafacture/triples/TripleSort.java | 4 + .../metafacture/triples/TripleSortTest.java | 82 ++++++++++++++++ 3 files changed, 126 insertions(+), 53 deletions(-) diff --git a/metafacture-triples/src/main/java/org/metafacture/triples/AbstractTripleSort.java b/metafacture-triples/src/main/java/org/metafacture/triples/AbstractTripleSort.java index 1ccd21710..6b98f295a 100644 --- a/metafacture-triples/src/main/java/org/metafacture/triples/AbstractTripleSort.java +++ b/metafacture-triples/src/main/java/org/metafacture/triples/AbstractTripleSort.java @@ -30,12 +30,14 @@ import java.util.Comparator; import java.util.List; import java.util.PriorityQueue; +import java.util.function.Function; /** * @author markus geipel * */ public abstract class AbstractTripleSort extends DefaultObjectPipe> implements MemoryWarningSystem.Listener { + /** * specifies the comparator */ @@ -63,17 +65,16 @@ public int order(final int indicator) { public abstract int order(int indicator); } - private final List buffer = new ArrayList(); + private final List buffer = new ArrayList<>(); private final List tempFiles; private Compare compare = Compare.SUBJECT; private Order order = Order.INCREASING; + private boolean numeric; private volatile boolean memoryLow; public AbstractTripleSort() { MemoryWarningSystem.addListener(this); - tempFiles = new ArrayList(); // Initialized here to let the - // compiler enforce the call to - // super() in subclasses. + tempFiles = new ArrayList<>(); // Initialized here to let the compiler enforce the call to super() in subclasses. } @Override @@ -93,6 +94,10 @@ protected final void setSortOrder(final Order newOrder) { order = newOrder; } + protected final void setSortNumeric(final boolean newNumeric) { + numeric = newNumeric; + } + @Override public final void process(final Triple namedValue) { if (memoryLow) { @@ -112,47 +117,38 @@ public final void process(final Triple namedValue) { } private void nextBatch() throws IOException { - Collections.sort(buffer, createComparator(compare, order)); + Collections.sort(buffer, createComparator()); final File tempFile = File.createTempFile("sort", "namedValues", null); tempFile.deleteOnExit(); - final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(tempFile)); - try { + try (ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(tempFile))) { for (final Triple triple : buffer) { triple.write(out); } } - finally { - out.close(); - } + buffer.clear(); tempFiles.add(tempFile); } @Override public final void onCloseStream() { - if (tempFiles.isEmpty()) { - Collections.sort(buffer, createComparator(compare, order)); + Collections.sort(buffer, createComparator()); + for (final Triple triple : buffer) { sortedTriple(triple); } + onFinished(); } else { - final Comparator comparator = createComparator(compare, order); - final PriorityQueue queue = new PriorityQueue(11, - new Comparator() { - // private final Comparator comparator = - // getComparator(); - - @Override - public int compare(final SortedTripleFileFacade o1, final SortedTripleFileFacade o2) { - return comparator.compare(o1.peek(), o2.peek()); - } - }); + final Comparator comparator = createComparator(); + final PriorityQueue queue = new PriorityQueue<>(11, (o1, o2) -> comparator.compare(o1.peek(), o2.peek())); + try { nextBatch(); + for (final File file : tempFiles) { queue.add(new SortedTripleFileFacade(file)); } @@ -168,6 +164,7 @@ public int compare(final SortedTripleFileFacade o1, final SortedTripleFileFacade queue.add(sortedFileFacade); } } + onFinished(); } catch (final IOException e) { @@ -179,6 +176,7 @@ public int compare(final SortedTripleFileFacade o1, final SortedTripleFileFacade } } } + MemoryWarningSystem.removeListener(this); } @@ -189,58 +187,47 @@ protected void onFinished() { protected abstract void sortedTriple(Triple namedValue); public final Comparator createComparator() { - return createComparator(compare, order); + return createComparator(compare, order, numeric); + } + + public static Comparator createComparator(final Compare compare, final Order order) { + return createComparator(compare, order, false); } - public static Comparator createComparator(final Compare compareBy, final Order order) { - final Comparator comparator; - switch (compareBy) { + private static Comparator createComparator(final Compare compare, final Order order, final boolean numeric) { + final Function tripleFunction; + switch (compare) { case ALL: - comparator = new Comparator() { - @Override - public int compare(final Triple o1, final Triple o2) { - return order.order(o1.compareTo(o2)); - } - }; - break; + return (o1, o2) -> order.order(o1.compareTo(o2)); case OBJECT: - comparator = new Comparator() { - @Override - public int compare(final Triple o1, final Triple o2) { - return order.order(o1.getObject().compareTo(o2.getObject())); - } - }; + tripleFunction = Triple::getObject; break; case SUBJECT: - comparator = new Comparator() { - @Override - public int compare(final Triple o1, final Triple o2) { - return order.order(o1.getSubject().compareTo(o2.getSubject())); - } - }; + tripleFunction = Triple::getSubject; break; case PREDICATE: default: - comparator = new Comparator() { - @Override - public int compare(final Triple o1, final Triple o2) { - return order.order(o1.getPredicate().compareTo(o2.getPredicate())); - } - }; + tripleFunction = Triple::getPredicate; break; } - return comparator; + final Function numericFunction = tripleFunction.andThen(Integer::valueOf); + return numeric ? + (o1, o2) -> order.order(numericFunction.apply(o1).compareTo(numericFunction.apply(o2))) : + (o1, o2) -> order.order(tripleFunction.apply(o1).compareTo(tripleFunction.apply(o2))); } @Override public final void onResetStream() { buffer.clear(); + for (final File file : tempFiles) { if (file.exists()) { file.delete(); } } + tempFiles.clear(); } + } diff --git a/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java b/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java index f563b3b31..b3f51ce48 100644 --- a/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java +++ b/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java @@ -49,4 +49,8 @@ public void setOrder(final Order order) { setSortOrder(order); } + public void setNumeric(final boolean numeric) { + setSortNumeric(numeric); + } + } diff --git a/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java b/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java index 00b2b2c27..956570a79 100644 --- a/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java +++ b/metafacture-triples/src/test/java/org/metafacture/triples/TripleSortTest.java @@ -221,6 +221,88 @@ public void shouldNotSortNumerically() { ); } + @Test + public void issue380_shouldOptionallySortNumericallyByIncreasingSubject() { + assertSort( + t -> { + t.setNumeric(true); + }, + "10 p1 o2", + "2 p1 o0", + "0 p1 o1", + "101 p0 o2", + "11 p2 o1", + // + "0 p1 o1", + "2 p1 o0", + "10 p1 o2", + "11 p2 o1", + "101 p0 o2" + ); + } + + @Test + public void issue380_shouldOptionallySortNumericallyByDecreasingSubject() { + assertSort( + t -> { + t.setNumeric(true); + t.setOrder(AbstractTripleSort.Order.DECREASING); + }, + "10 p1 o2", + "2 p1 o0", + "0 p1 o1", + "101 p0 o2", + "11 p2 o1", + // + "101 p0 o2", + "11 p2 o1", + "10 p1 o2", + "2 p1 o0", + "0 p1 o1" + ); + } + + @Test(expected=NumberFormatException.class) + public void shouldFailToSortNumericallyWithInvalidNumber() { + assertSort( + t -> { + t.setNumeric(true); + }, + "10 p1 o2", + "2 p1 o0", + "0 p1 o1", + "1x1 p0 o2", + "11 p2 o1", + // + "", + "", + "", + "", + "" + ); + } + + @Test + public void shouldNotSortNumericallyByTriple() { + assertSort( + t -> { + t.setNumeric(true); + t.setBy(AbstractTripleSort.Compare.ALL); + }, + "10 p1 o2", + "2 p1 o0", + "0 p1 o1", + "101 p0 o2", + "11 p2 o1", + // + "0 p1 o1", + "10 p1 o2", + "101 p0 o2", + "11 p2 o1", + "2 p1 o0" + ); + } + public void assertSort(final Consumer consumer, final String... triples) { final BiConsumer> processor = (i, c) -> { final int j = triples.length / 2; From 26aeeaf25e94af7a44bb31c1c1a2d3c2a2cf9e4b Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Fri, 15 Oct 2021 16:33:24 +0200 Subject: [PATCH 3/3] Improve description Will be used to update the flux-commands.md. See #380. --- .../src/main/java/org/metafacture/triples/TripleSort.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java b/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java index b3f51ce48..33e24468e 100644 --- a/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java +++ b/metafacture-triples/src/main/java/org/metafacture/triples/TripleSort.java @@ -27,7 +27,7 @@ * @author markus geipel * */ -@Description("Sorts triples") +@Description("Sorts triples. Several options can be combined, e.g. `by=\"object\",numeric=\"true\",order=\"decreasing\"` will numerically sort the Object of the triples in decreasing order (given that all Objects are indeed of numeric type).") @In(Triple.class) @Out(Triple.class) @FluxCommand("sort-triples")