From f9a952e1583182a936a5512f35d87aaaca408e0d Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Tue, 2 Jun 2020 15:23:53 +0200 Subject: [PATCH 1/3] Introduce Metamorph 1.1 With this version of Metamorph entities (well, entity events) can be passed through. It is slightly incomaptible with the default Metamorph where two tests would fail: - org.metafacture.metamorph.collectors.EntityTest > shouldEmitEntityOnEachFlushEvent - org.metafacture.metamorph.functions.UniqueTest -> shouldAllowSelectingTheUniqueScope So this introduces a "" element under the "" element in morph. The data is flattened, as with Metamorph 1, but the entity's "start" and "end" events are passed through so that the receiver can handle the flattened data structure, unflatten it etc. By preserving the entity events it's now also possible, without any workarounds, to handle reiterations of entities having the same name. - add "version" element to metamorph.xsd See https://github.com/metafacture/metafacture-core/issues/107. See also https://github.com/hagbeck/metafacture-sandbox/tree/master/enrich_marcxml. --- .../src/main/resources/schemata/metamorph.xsd | 6 +- .../org/metafacture/metamorph/Metamorph.java | 7 ++- .../src/main/resources/schemata/metamorph.xsd | 2 + .../metafacture/metamorph/MetamorphTest.java | 60 ++++++++++++++++++- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/metamorph-test/src/main/resources/schemata/metamorph.xsd b/metamorph-test/src/main/resources/schemata/metamorph.xsd index 52c04d2c4..f314f7f9a 100644 --- a/metamorph-test/src/main/resources/schemata/metamorph.xsd +++ b/metamorph-test/src/main/resources/schemata/metamorph.xsd @@ -54,12 +54,14 @@ + + @@ -302,9 +304,9 @@ + default="false" /> + default="false" /> diff --git a/metamorph/src/main/java/org/metafacture/metamorph/Metamorph.java b/metamorph/src/main/java/org/metafacture/metamorph/Metamorph.java index 2a141c7fa..ae064d9df 100644 --- a/metamorph/src/main/java/org/metafacture/metamorph/Metamorph.java +++ b/metamorph/src/main/java/org/metafacture/metamorph/Metamorph.java @@ -268,8 +268,8 @@ public void startEntity(final String name) { entityCountStack.push(Integer.valueOf(entityCount)); flattener.startEntity(name); - - + if (maps.containsKey(METADATA) && maps.get(METADATA).getOrDefault("version","").equals("1.1")) + outputStreamReceiver.startEntity(name); } @@ -278,7 +278,8 @@ public void endEntity() { dispatch(flattener.getCurrentPath(), "", null); currentEntityCount = entityCountStack.pop().intValue(); flattener.endEntity(); - + if (maps.containsKey(METADATA) && maps.get(METADATA).getOrDefault("version","").equals("1.1")) + outputStreamReceiver.endEntity(); } diff --git a/metamorph/src/main/resources/schemata/metamorph.xsd b/metamorph/src/main/resources/schemata/metamorph.xsd index 9bbea22db..832abc480 100644 --- a/metamorph/src/main/resources/schemata/metamorph.xsd +++ b/metamorph/src/main/resources/schemata/metamorph.xsd @@ -54,12 +54,14 @@ + + diff --git a/metamorph/src/test/java/org/metafacture/metamorph/MetamorphTest.java b/metamorph/src/test/java/org/metafacture/metamorph/MetamorphTest.java index 8e0715229..f94624287 100644 --- a/metamorph/src/test/java/org/metafacture/metamorph/MetamorphTest.java +++ b/metamorph/src/test/java/org/metafacture/metamorph/MetamorphTest.java @@ -21,6 +21,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -30,9 +31,11 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.metafacture.framework.StreamReceiver; import org.metafacture.framework.helpers.DefaultStreamReceiver; import org.metafacture.metamorph.api.Maps; import org.metafacture.metamorph.api.NamedValueReceiver; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @@ -45,13 +48,15 @@ * @author Christoph Böhme (rewrite) */ public final class MetamorphTest { - @Rule public MockitoRule mockitoRule = MockitoJUnit.rule(); @Mock private NamedValueReceiver namedValueReceiver; + @Mock + private StreamReceiver receiver; + private Metamorph metamorph; @Before @@ -189,4 +194,57 @@ public void shouldThrowIllegalStateExceptionIfEntityIsNotClosed() { metamorph.endRecord(); // Exception expected } + @Test + public void metamorph1() { + metamorph = InlineMorph.in(this) // + .with("") // + .with(" ")// + .with("")// + .createConnectedTo(receiver); + + metamorph.startRecord("1"); + metamorph.startEntity("clone"); + metamorph.literal("id", "0"); + metamorph.endEntity(); + metamorph.startEntity("clone"); + metamorph.literal("id", "1"); + metamorph.endEntity(); + metamorph.endRecord(); + + final InOrder ordered = inOrder(receiver); + ordered.verify(receiver).startRecord("1"); + ordered.verify(receiver).literal("clone.id", "0"); + ordered.verify(receiver).literal("clone.id", "1"); + ordered.verify(receiver).endRecord(); + } + + @Test + public void metamorph1_1() { + metamorph = InlineMorph.in(this).with("") // + .with("1.1")// + .with("")// + .with("")// + .with(" ")// + .with("")// + .createConnectedTo(receiver); + + metamorph.startRecord("1"); + metamorph.startEntity("clone"); + metamorph.literal("id", "0"); + metamorph.endEntity(); + metamorph.startEntity("clone"); + metamorph.literal("id", "1"); + metamorph.endEntity(); + metamorph.endRecord(); + + final InOrder ordered = inOrder(receiver); + ordered.verify(receiver).startRecord("1"); + ordered.verify(receiver).startEntity("clone"); + ordered.verify(receiver).literal("clone.id", "0"); + ordered.verify(receiver).endEntity(); + ordered.verify(receiver).startEntity("clone"); + ordered.verify(receiver).literal("clone.id", "1"); + ordered.verify(receiver).endEntity(); + ordered.verify(receiver).endRecord(); + } } From 7ca5615937b8336dd00059baf61bf99ae482d0b5 Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Tue, 2 Jun 2020 17:47:59 +0200 Subject: [PATCH 2/3] Improve MarcXmlEncoer to work also with Metamorph 1.1 --- .../biblio/marc21/MarcXmlEncoder.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java b/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java index 9fe831575..d67cfff52 100644 --- a/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java +++ b/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java @@ -144,10 +144,10 @@ public void startEntity(final String name) { currentEntity = name; if (!name.equals(Marc21EventNames.LEADER_ENTITY)) { if (name.length() != 5) { - String message = String.format("Entity too short." + "Got a string ('%s') of length %d." - + "Expected a length of 5 (field + indicators).", name, name.length()); - throw new MetafactureException(message); - } + String message = String.format("Entity too short." + "Got a string ('%s') of length %d." + + "Expected a length of 5 (field + indicators).", name, name.length()); + throw new MetafactureException(message); + } String tag = name.substring(0, 3); String ind1 = name.substring(3, 4); @@ -156,7 +156,10 @@ public void startEntity(final String name) { writeRaw(String.format(DATAFIELD_OPEN_TEMPLATE, tag, ind1, ind2)); prettyPrintNewLine(); incrementIndentationLevel(); - } + } else { prettyPrintIndentation(); + writeRaw(LEADER_OPEN_TEMPLATE); + } + } @Override @@ -172,22 +175,23 @@ public void endEntity() { @Override public void literal(final String name, final String value) { + if (value == null || value.isEmpty()) + return; if (currentEntity.equals("")) { prettyPrintIndentation(); - writeRaw(String.format(CONTROLFIELD_OPEN_TEMPLATE, name)); + writeRaw(String.format(CONTROLFIELD_OPEN_TEMPLATE, name.replaceFirst("\\W",""))); writeEscaped(value.trim()); writeRaw(CONTROLFIELD_CLOSE); prettyPrintNewLine(); } else if (!currentEntity.equals(Marc21EventNames.LEADER_ENTITY)) { prettyPrintIndentation(); - writeRaw(String.format(SUBFIELD_OPEN_TEMPLATE, name)); + writeRaw(String.format(SUBFIELD_OPEN_TEMPLATE, name.charAt(name.indexOf('.')+1))); writeEscaped(value.trim()); writeRaw(SUBFIELD_CLOSE); prettyPrintNewLine(); } else { - if (name.equals(Marc21EventNames.LEADER_ENTITY)) { - prettyPrintIndentation(); - writeRaw(LEADER_OPEN_TEMPLATE + value + LEADER_CLOSE_TEMPLATE); + { + writeRaw(value + LEADER_CLOSE_TEMPLATE); prettyPrintNewLine(); } } @@ -241,8 +245,8 @@ private void writeEscaped(final String str) { private void prettyPrintIndentation() { if (formatted) { - String prefix = String.join("", Collections.nCopies(indentationLevel, INDENT)); - builder.append(prefix); + String prefix = String.join("", Collections.nCopies(indentationLevel, INDENT)); + builder.append(prefix); } } @@ -256,4 +260,4 @@ private void sendAndClearData() { getReceiver().process(builder.toString()); builder.delete(0, builder.length()); } -} \ No newline at end of file +} From 7011a2a4c337735ae09f7b620f1f8e05764c5ada Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Tue, 2 Jun 2020 18:17:18 +0200 Subject: [PATCH 3/3] Reformat MarcXmlEncoder --- .../biblio/marc21/MarcXmlEncoder.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java b/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java index d67cfff52..5b9ad3dad 100644 --- a/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java +++ b/metafacture-biblio/src/main/java/org/metafacture/biblio/marc21/MarcXmlEncoder.java @@ -103,8 +103,7 @@ public void setXmlEncoding(String xmlEncoding) { /** * Formats the resulting xml, by indentation. * - * @param formatted - * True, if formatting is activated. + * @param formatted True, if formatting is activated. */ public void setFormatted(boolean formatted) { this.formatted = formatted; @@ -144,10 +143,10 @@ public void startEntity(final String name) { currentEntity = name; if (!name.equals(Marc21EventNames.LEADER_ENTITY)) { if (name.length() != 5) { - String message = String.format("Entity too short." + "Got a string ('%s') of length %d." - + "Expected a length of 5 (field + indicators).", name, name.length()); - throw new MetafactureException(message); - } + String message = String.format("Entity too short." + "Got a string ('%s') of length %d." + + "Expected a length of 5 (field + indicators).", name, name.length()); + throw new MetafactureException(message); + } String tag = name.substring(0, 3); String ind1 = name.substring(3, 4); @@ -156,9 +155,10 @@ public void startEntity(final String name) { writeRaw(String.format(DATAFIELD_OPEN_TEMPLATE, tag, ind1, ind2)); prettyPrintNewLine(); incrementIndentationLevel(); - } else { prettyPrintIndentation(); - writeRaw(LEADER_OPEN_TEMPLATE); - } + } else { + prettyPrintIndentation(); + writeRaw(LEADER_OPEN_TEMPLATE); + } } @@ -176,22 +176,22 @@ public void endEntity() { @Override public void literal(final String name, final String value) { if (value == null || value.isEmpty()) - return; + return; if (currentEntity.equals("")) { prettyPrintIndentation(); - writeRaw(String.format(CONTROLFIELD_OPEN_TEMPLATE, name.replaceFirst("\\W",""))); + writeRaw(String.format(CONTROLFIELD_OPEN_TEMPLATE, name.replaceFirst("\\W", ""))); writeEscaped(value.trim()); writeRaw(CONTROLFIELD_CLOSE); prettyPrintNewLine(); } else if (!currentEntity.equals(Marc21EventNames.LEADER_ENTITY)) { prettyPrintIndentation(); - writeRaw(String.format(SUBFIELD_OPEN_TEMPLATE, name.charAt(name.indexOf('.')+1))); + writeRaw(String.format(SUBFIELD_OPEN_TEMPLATE, name.charAt(name.indexOf('.') + 1))); writeEscaped(value.trim()); writeRaw(SUBFIELD_CLOSE); prettyPrintNewLine(); } else { { - writeRaw(value + LEADER_CLOSE_TEMPLATE); + writeRaw(value + LEADER_CLOSE_TEMPLATE); prettyPrintNewLine(); } } @@ -245,8 +245,8 @@ private void writeEscaped(final String str) { private void prettyPrintIndentation() { if (formatted) { - String prefix = String.join("", Collections.nCopies(indentationLevel, INDENT)); - builder.append(prefix); + String prefix = String.join("", Collections.nCopies(indentationLevel, INDENT)); + builder.append(prefix); } }