From b8a51b2b4199fe0218514af49b4d0eb561baa070 Mon Sep 17 00:00:00 2001 From: ghik Date: Thu, 29 Mar 2018 17:59:06 +0200 Subject: [PATCH 1/4] relaxed field order preservation requirement for ObjectInput --- .../commons/serialization/InputOutput.scala | 37 ++++++++----- .../SimpleValueInputOutput.scala | 6 ++- .../commons/serialization/macroCodecs.scala | 53 ++++++++++--------- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala index 78c31f72e..8e80ed118 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala @@ -1,8 +1,6 @@ package com.avsystem.commons package serialization -import com.avsystem.commons.serialization.GenCodec.ReadFailure - /** * Represents an abstract sink to which a value may be serialized (written). * An [[Output]] instance should be assumed to be stateful. After calling any of the `write` methods, it MUST NOT be @@ -192,12 +190,34 @@ trait ObjectInput extends Any with SequentialInput { self => * You MUST NOT call `nextField()` again until this [[com.avsystem.commons.serialization.FieldInput FieldInput]] * is fully read or skipped. *

- * Subsequent invocations of `nextField` MUST return fields in exactly the same order as they were written - * using [[com.avsystem.commons.serialization.ObjectOutput.writeField ObjectOutput.writeField]]. - * In other words, serialization format MUST preserve order of object fields. + * Serialization format implemented by this [[ObjectInput]] must either preserve order of fields (as they are + * written by corresponding [[ObjectOutput]]) OR it must provide random field access capability. + * */ def nextField(): FieldInput + /** + * If serialization format implemented by [[ObjectInput]] does NOT preserve field order, then this method MUST + * be overridden to support random field access. It should return non-empty [[Opt]] containing input for every field + * present in the object, regardless of field order assumed by [[nextField()]]. + * [[Opt.Empty]] is returned when field is absent or always when this [[ObjectInput]] does not support random field + * access (in which case it must preserve field order instead). + * NOTE: calling [[peekField]] and using [[FieldInput]] returned by it MUST NOT in any way influence results + * returned by [[nextField()]] and [[hasNext]]. For example, if a [[FieldInput]] for particular field has already been + * accessed using [[peekField()]] but has not yet been returned by [[nextField()]] then it MUST be returned at some + * point in the future by [[nextField()]]. + */ + def peekField(name: String): Opt[FieldInput] = Opt.Empty + def skipRemaining() = while (hasNext) nextField().skip() def iterator[A](readFun: Input => A): Iterator[(String, A)] = new Iterator[(String, A)] { @@ -214,11 +234,4 @@ trait ObjectInput extends Any with SequentialInput { self => */ trait FieldInput extends Input { def fieldName: String - - def assertField(expectedName: String): this.type = { - if (fieldName != expectedName) { - throw new ReadFailure(s"Expected $expectedName as next field, got $fieldName") - } - this - } } diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/SimpleValueInputOutput.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/SimpleValueInputOutput.scala index fdcc28c2a..031aa1ad0 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/SimpleValueInputOutput.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/SimpleValueInputOutput.scala @@ -55,7 +55,7 @@ class SimpleValueOutput(consumer: Any => Unit) extends Output { def writeBoolean(boolean: Boolean) = consumer(boolean) def writeObject() = new ObjectOutput { - private val result = new mutable.LinkedHashMap[String, Any] + private val result = new mutable.HashMap[String, Any] def writeField(key: String) = new SimpleValueOutput(v => result += ((key, v))) def finish() = consumer(result) } @@ -95,10 +95,12 @@ class SimpleValueInput(value: Any) extends Input { def readNull() = if (value == null) null else throw new ReadFailure("not null") def readObject() = new ObjectInput { - private val it = doRead[BMap[String, Any]].iterator.map { + private val map = doRead[BMap[String, Any]] + private val it = map.iterator.map { case (k, v) => new SimpleValueFieldInput(k, v) } def nextField() = it.next() + override def peekField(name: String) = map.getOpt(name).map(new SimpleValueFieldInput(name, _)) def hasNext = it.hasNext } diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala index 1cdc47e3a..8a75742b8 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala @@ -143,38 +143,43 @@ abstract class FlatSealedHierarchyCodec[T]( final def readObject(input: ObjectInput): T = { val oooFields = new FieldValues(oooFieldNames, oooDeps, typeRepr) - def read(): T = - if (input.hasNext) { - val fi = input.nextField() - if (fi.fieldName == caseFieldName) { - val caseName = readCaseName(fi) - caseIndexByName(caseName) match { - case -1 => unknownCase(caseName) - case idx => readFlatCase(caseName, oooFields, input, caseDeps(idx)) - } - } else if (!oooFields.tryReadField(fi)) { - if (caseDependentFieldNames.contains(fi.fieldName)) { - if (defaultCaseIdx != -1) { - val defaultCaseName = caseNames(defaultCaseIdx) - val wrappedInput = new DefaultCaseObjectInput(fi, input, defaultCaseName) - readFlatCase(defaultCaseName, oooFields, wrappedInput, caseDeps(defaultCaseIdx)) - } else { - missingCase(fi.fieldName) - } + def readCase(caseNameField: FieldInput): T = { + val caseName = readCaseName(caseNameField) + caseIndexByName(caseName) match { + case -1 => unknownCase(caseName) + case idx => readFlatCase(caseName, oooFields, input, caseDeps(idx)) + } + } + + def read(): T = if (input.hasNext) { + val fi = input.nextField() + if (fi.fieldName == caseFieldName) readCase(fi) + else if (!oooFields.tryReadField(fi)) { + if (caseDependentFieldNames.contains(fi.fieldName)) { + if (defaultCaseIdx != -1) { + val defaultCaseName = caseNames(defaultCaseIdx) + val wrappedInput = new DefaultCaseObjectInput(fi, input, defaultCaseName) + readFlatCase(defaultCaseName, oooFields, wrappedInput, caseDeps(defaultCaseIdx)) } else { - fi.skip() - read() + missingCase(fi.fieldName) } } else { + fi.skip() read() } - } else if (defaultCaseIdx != -1) { - readFlatCase(caseNames(defaultCaseIdx), oooFields, input, caseDeps(defaultCaseIdx)) } else { - missingCase + read() } + } else if (defaultCaseIdx != -1) { + readFlatCase(caseNames(defaultCaseIdx), oooFields, input, caseDeps(defaultCaseIdx)) + } else { + missingCase + } - read() + input.peekField(caseFieldName) match { + case Opt(fi) => readCase(fi) + case Opt.Empty => read() + } } } From 540eee04339cd776d1b46ae3de0e4057f673d2d7 Mon Sep 17 00:00:00 2001 From: ghik Date: Tue, 3 Apr 2018 11:03:59 +0200 Subject: [PATCH 2/4] docs and tests updated --- .../commons/serialization/InputOutput.scala | 25 +++--- .../commons/serialization/GenCodecTest.scala | 90 ++++++++++--------- docs/GenCodec.md | 50 +++++++++-- 3 files changed, 106 insertions(+), 59 deletions(-) diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala index 8e80ed118..6b3309778 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/InputOutput.scala @@ -190,31 +190,32 @@ trait ObjectInput extends Any with SequentialInput { self => * You MUST NOT call `nextField()` again until this [[com.avsystem.commons.serialization.FieldInput FieldInput]] * is fully read or skipped. *

- * Serialization format implemented by this [[ObjectInput]] must either preserve order of fields (as they are - * written by corresponding [[ObjectOutput]]) OR it must provide random field access capability. + * Serialization format implemented by this `ObjectInput` must either preserve order of fields (as they are + * written by corresponding `ObjectOutput`) OR it must provide random field access capability. * */ def nextField(): FieldInput /** - * If serialization format implemented by [[ObjectInput]] does NOT preserve field order, then this method MUST + * If serialization format implemented by `ObjectInput` does NOT preserve field order, then this method MUST * be overridden to support random field access. It should return non-empty [[Opt]] containing input for every field - * present in the object, regardless of field order assumed by [[nextField()]]. - * [[Opt.Empty]] is returned when field is absent or always when this [[ObjectInput]] does not support random field + * present in the object, regardless of field order assumed by [[nextField]]. + * `Opt.Empty` is returned when field is absent or always when this `ObjectInput` does not support random field * access (in which case it must preserve field order instead). - * NOTE: calling [[peekField]] and using [[FieldInput]] returned by it MUST NOT in any way influence results - * returned by [[nextField()]] and [[hasNext]]. For example, if a [[FieldInput]] for particular field has already been - * accessed using [[peekField()]] but has not yet been returned by [[nextField()]] then it MUST be returned at some - * point in the future by [[nextField()]]. + * NOTE: calling [[peekField]] and using [[FieldInput]] returned by it MUST NOT change state of this `ObjectInput`. + * Therefore, it cannot in any way influence results returned by [[nextField]] and [[hasNext]]. + * For example, if a [[FieldInput]] for particular field has already been + * accessed using [[peekField]] but has not yet been returned by [[nextField]] then it MUST be returned at some + * point in the future by [[nextField]]. */ def peekField(name: String): Opt[FieldInput] = Opt.Empty diff --git a/commons-core/src/test/scala/com/avsystem/commons/serialization/GenCodecTest.scala b/commons-core/src/test/scala/com/avsystem/commons/serialization/GenCodecTest.scala index 2bd5b6de1..3da6d5b20 100644 --- a/commons-core/src/test/scala/com/avsystem/commons/serialization/GenCodecTest.scala +++ b/commons-core/src/test/scala/com/avsystem/commons/serialization/GenCodecTest.scala @@ -70,10 +70,10 @@ class GenCodecTest extends CodecTestBase { testWriteReadAndAutoWriteRead[JSortedSet[Int]](jTreeSet, List(1, 2, 3)) testWriteReadAndAutoWriteRead[JNavigableSet[Int]](jTreeSet, List(1, 2, 3)) testWriteReadAndAutoWriteRead[JTreeSet[Int]](jTreeSet, List(1, 2, 3)) - testWriteReadAndAutoWriteRead[JMap[String, Int]](jHashMap, ListMap("1" -> 1, "2" -> 2, "3" -> 3)) - testWriteReadAndAutoWriteRead[JHashMap[String, Int]](jHashMap, ListMap("1" -> 1, "2" -> 2, "3" -> 3)) - testWriteReadAndAutoWriteRead[JLinkedHashMap[String, Int]](jLinkedHashMap, ListMap("1" -> 1, "2" -> 2, "3" -> 3)) - testWriteReadAndAutoWriteRead[JHashMap[Int, Int]](jIntHashMap, ListMap("1" -> 1, "2" -> 2, "3" -> 3)) + testWriteReadAndAutoWriteRead[JMap[String, Int]](jHashMap, Map("1" -> 1, "2" -> 2, "3" -> 3)) + testWriteReadAndAutoWriteRead[JHashMap[String, Int]](jHashMap, Map("1" -> 1, "2" -> 2, "3" -> 3)) + testWriteReadAndAutoWriteRead[JLinkedHashMap[String, Int]](jLinkedHashMap, Map("1" -> 1, "2" -> 2, "3" -> 3)) + testWriteReadAndAutoWriteRead[JHashMap[Int, Int]](jIntHashMap, Map("1" -> 1, "2" -> 2, "3" -> 3)) } test("NoState test") { @@ -83,12 +83,12 @@ class GenCodecTest extends CodecTestBase { test("collections and wrappers test") { testWriteReadAndAutoWriteRead[Option[Int]](option, List(42)) - testWriteReadAndAutoWriteRead[Either[Int, String]](Left(42), ListMap("Left" -> 42)) - testWriteReadAndAutoWriteRead[Either[Int, String]](Right("lol"), ListMap("Right" -> "lol")) + testWriteReadAndAutoWriteRead[Either[Int, String]](Left(42), Map("Left" -> 42)) + testWriteReadAndAutoWriteRead[Either[Int, String]](Right("lol"), Map("Right" -> "lol")) testWriteReadAndAutoWriteRead[List[Int]](list, list) testWriteReadAndAutoWriteRead[Set[Int]](set, set.toList) testWriteReadAndAutoWriteRead[Map[String, Int]](map, map) - testWriteReadAndAutoWriteRead[Map[Int, Int]](intMap, ListMap("1" -> 1, "2" -> 2, "3" -> 3)) + testWriteReadAndAutoWriteRead[Map[Int, Int]](intMap, Map("1" -> 1, "2" -> 2, "3" -> 3)) testWriteReadAndAutoWriteRead[IHashMap[String, Int]](hashMap, hashMap) } @@ -122,7 +122,7 @@ class GenCodecTest extends CodecTestBase { object SingleArgCaseClass extends HasGenCodec[SingleArgCaseClass] test("single arg case class test") { - testWriteReadAndAutoWriteRead(SingleArgCaseClass("something"), ListMap("str" -> "something")) + testWriteReadAndAutoWriteRead(SingleArgCaseClass("something"), Map("str" -> "something")) } @transparent @@ -149,7 +149,7 @@ class GenCodecTest extends CodecTestBase { test("case class test") { testWriteReadAndAutoWriteRead(SomeCaseClass("dafuq", List(1, 2, 3)), - ListMap("some.str" -> "dafuq", "intList" -> List(1, 2, 3), "someStrLen" -> 5) + Map("some.str" -> "dafuq", "intList" -> List(1, 2, 3), "someStrLen" -> 5) ) } @@ -166,7 +166,7 @@ class GenCodecTest extends CodecTestBase { } test("case class with wildcard test") { - testWriteReadAndAutoWriteRead(CaseClassWithWildcard(Stuff("lol")), ListMap("stuff" -> "lol")) + testWriteReadAndAutoWriteRead(CaseClassWithWildcard(Stuff("lol")), Map("stuff" -> "lol")) } class CaseClassLike(val str: String, val intList: List[Int]) @@ -179,7 +179,7 @@ class GenCodecTest extends CodecTestBase { test("case class like test") { testWriteReadAndAutoWriteRead(CaseClassLike("dafuq", List(1, 2, 3)), - ListMap("some.str" -> "dafuq", "intList" -> List(1, 2, 3)) + Map("some.str" -> "dafuq", "intList" -> List(1, 2, 3)) ) } @@ -199,7 +199,7 @@ class GenCodecTest extends CodecTestBase { test("case class like with inherited apply/unapply test") { testWriteReadAndAutoWriteRead(HasInheritedApply("dafuq", List(1, 2, 3)), - ListMap("a" -> "dafuq", "lb" -> List(1, 2, 3)) + Map("a" -> "dafuq", "lb" -> List(1, 2, 3)) ) } @@ -212,7 +212,7 @@ class GenCodecTest extends CodecTestBase { test("apply/unapply provider based codec test") { implicit val tpCodec: GenCodec[ThirdParty] = GenCodec.fromApplyUnapplyProvider[ThirdParty](ThirdPartyFakeCompanion) testWriteReadAndAutoWriteRead(ThirdParty(42, "lol"), - ListMap("str" -> "lol", "int" -> 42) + Map("str" -> "lol", "int" -> 42) ) } @@ -223,7 +223,7 @@ class GenCodecTest extends CodecTestBase { test("varargs case class test") { testWriteReadAndAutoWriteRead(VarargsCaseClass(42, "foo", "bar"), - ListMap("int" -> 42, "strings" -> List("foo", "bar")) + Map("int" -> 42, "strings" -> List("foo", "bar")) ) } @@ -237,7 +237,7 @@ class GenCodecTest extends CodecTestBase { test("varargs case class like test") { testWriteReadAndAutoWriteRead(VarargsCaseClassLike("dafuq", 1, 2, 3), - ListMap("some.str" -> "dafuq", "ints" -> List(1, 2, 3)) + Map("some.str" -> "dafuq", "ints" -> List(1, 2, 3)) ) } @@ -247,9 +247,9 @@ class GenCodecTest extends CodecTestBase { } test("case class with default values test") { - testWriteReadAndAutoWriteRead(HasDefaults(str = "lol"), ListMap("str" -> "lol")) - testWriteReadAndAutoWriteRead(HasDefaults(43, "lol"), ListMap("int" -> 43, "str" -> "lol")) - testWriteReadAndAutoWriteRead(HasDefaults(str = null), ListMap("str" -> null)) + testWriteReadAndAutoWriteRead(HasDefaults(str = "lol"), Map("str" -> "lol")) + testWriteReadAndAutoWriteRead(HasDefaults(43, "lol"), Map("int" -> 43, "str" -> "lol")) + testWriteReadAndAutoWriteRead(HasDefaults(str = null), Map("str" -> null)) } case class Node[T](value: T, children: List[Node[T]] = Nil) @@ -287,41 +287,47 @@ class GenCodecTest extends CodecTestBase { } test("recursively defined sealed hierarchy with explicit case class codec test") { - testWriteReadAndAutoWriteRead[CustomList](CustomTail, ListMap("CustomTail" -> Map())) + testWriteReadAndAutoWriteRead[CustomList](CustomTail, Map("CustomTail" -> Map())) testWriteReadAndAutoWriteRead[CustomList](CustomCons(CustomCons(CustomTail)), - ListMap("CustomCons" -> ListMap("CustomCons" -> ListMap("CustomTail" -> Map())))) + Map("CustomCons" -> Map("CustomCons" -> Map("CustomTail" -> Map())))) } test("value class test") { - testWriteReadAndAutoWriteRead(ValueClass("costam"), ListMap("str" -> "costam")) + testWriteReadAndAutoWriteRead(ValueClass("costam"), Map("str" -> "costam")) } test("sealed hierarchy test") { testWriteReadAndAutoWriteRead[SealedBase](SealedBase.CaseObject, - ListMap("CaseObject" -> Map())) + Map("CaseObject" -> Map())) testWriteReadAndAutoWriteRead[SealedBase](SealedBase.CaseClass("fuu"), - ListMap("CaseClass" -> ListMap("str" -> "fuu"))) + Map("CaseClass" -> Map("str" -> "fuu"))) testWriteReadAndAutoWriteRead[SealedBase](SealedBase.InnerBase.InnerCaseObject, - ListMap("InnerCaseObject" -> Map())) + Map("InnerCaseObject" -> Map())) testWriteReadAndAutoWriteRead[SealedBase](SealedBase.InnerBase.InnerCaseClass("fuu"), - ListMap("InnerCaseClass" -> ListMap("str" -> "fuu"))) + Map("InnerCaseClass" -> Map("str" -> "fuu"))) } test("flat sealed hierarchy test") { testWriteReadAndAutoWriteRead[FlatSealedBase](FlatSealedBase.FirstCase("fuu", 42), - ListMap("_case" -> "FirstCase", "_id" -> "fuu", "int" -> 42, "upper_id" -> "FUU")) + Map("_case" -> "FirstCase", "_id" -> "fuu", "int" -> 42, "upper_id" -> "FUU")) testWriteReadAndAutoWriteRead[FlatSealedBase](FlatSealedBase.SecondCase("bar", 3.14, 1.0, 2.0), - ListMap("_case" -> "SecondCase", "_id" -> "bar", "dbl" -> 3.14, "moar" -> List(1.0, 2.0), "upper_id" -> "BAR")) + Map("_case" -> "SecondCase", "_id" -> "bar", "dbl" -> 3.14, "moar" -> List(1.0, 2.0), "upper_id" -> "BAR")) testWriteReadAndAutoWriteRead[FlatSealedBase](FlatSealedBase.ThirdCase, - ListMap("_case" -> "ThirdCase", "_id" -> "third", "upper_id" -> "THIRD")) + Map("_case" -> "ThirdCase", "_id" -> "third", "upper_id" -> "THIRD")) + } + + test("random field access dependent flat sealed hierarchy reading test") { + testReadAndAutoRead[FlatSealedBase]( + ListMap("_id" -> "fuu", "int" -> 42, "upper_id" -> "FUU", "_case" -> "FirstCase"), + FlatSealedBase.FirstCase("fuu", 42)) } test("out of order field in flat sealed hierarchy test") { testReadAndAutoRead[FlatSealedBase]( - ListMap("_id" -> "fuu", "upper_id" -> "FUU", "random" -> 13, "_case" -> "FirstCase", "int" -> 42), + Map("_id" -> "fuu", "upper_id" -> "FUU", "random" -> 13, "_case" -> "FirstCase", "int" -> 42), FlatSealedBase.FirstCase("fuu", 42)) testReadAndAutoRead[FlatSealedBase]( - ListMap("_id" -> "bar", "upper_id" -> "FUU", "random" -> 13, "_case" -> "SecondCase", "dbl" -> 3.14, "moar" -> List(1.0, 2.0)), + Map("_id" -> "bar", "upper_id" -> "FUU", "random" -> 13, "_case" -> "SecondCase", "dbl" -> 3.14, "moar" -> List(1.0, 2.0)), FlatSealedBase.SecondCase("bar", 3.14, 1.0, 2.0)) } @@ -342,10 +348,10 @@ class GenCodecTest extends CodecTestBase { } test("GADT test") { - testWriteReadAndAutoWriteRead[Expr[_]](NullExpr, ListMap("NullExpr" -> Map())) - testWriteReadAndAutoWriteRead[Expr[_]](StringExpr("stringzor"), ListMap("StringExpr" -> ListMap("str" -> "stringzor"))) - testWriteReadAndAutoWriteRead[Expr[String]](StringExpr("stringzor"), ListMap("StringExpr" -> ListMap("str" -> "stringzor"))) - testWriteReadAndAutoWriteRead[BaseExpr](StringExpr("stringzor"), ListMap("StringExpr" -> ListMap("str" -> "stringzor"))) + testWriteReadAndAutoWriteRead[Expr[_]](NullExpr, Map("NullExpr" -> Map())) + testWriteReadAndAutoWriteRead[Expr[_]](StringExpr("stringzor"), Map("StringExpr" -> Map("str" -> "stringzor"))) + testWriteReadAndAutoWriteRead[Expr[String]](StringExpr("stringzor"), Map("StringExpr" -> Map("str" -> "stringzor"))) + testWriteReadAndAutoWriteRead[BaseExpr](StringExpr("stringzor"), Map("StringExpr" -> Map("str" -> "stringzor"))) } sealed trait Tree[T] @@ -370,11 +376,11 @@ class GenCodecTest extends CodecTestBase { Leaf(3) ) ), - ListMap("Branch" -> Map( - "left" -> ListMap("Leaf" -> ListMap("value" -> 1)), - "right" -> ListMap("Branch" -> Map( - "left" -> ListMap("Leaf" -> ListMap("value" -> 2)), - "right" -> ListMap("Leaf" -> ListMap("value" -> 3)) + Map("Branch" -> Map( + "left" -> Map("Leaf" -> Map("value" -> 1)), + "right" -> Map("Branch" -> Map( + "left" -> Map("Leaf" -> Map("value" -> 2)), + "right" -> Map("Leaf" -> Map("value" -> 3)) )) )) ) @@ -391,9 +397,9 @@ class GenCodecTest extends CodecTestBase { } test("sealed enum test") { - testWriteReadAndAutoWriteRead[Enumz](Enumz.First, ListMap("Primary" -> Map())) - testWriteReadAndAutoWriteRead[Enumz](Enumz.Second, ListMap("Second" -> Map())) - testWriteReadAndAutoWriteRead[Enumz](Enumz.Third, ListMap("Third" -> Map())) + testWriteReadAndAutoWriteRead[Enumz](Enumz.First, Map("Primary" -> Map())) + testWriteReadAndAutoWriteRead[Enumz](Enumz.Second, Map("Second" -> Map())) + testWriteReadAndAutoWriteRead[Enumz](Enumz.Third, Map("Third" -> Map())) } sealed trait KeyEnumz diff --git a/docs/GenCodec.md b/docs/GenCodec.md index 219a9eda9..581ab599b 100644 --- a/docs/GenCodec.md +++ b/docs/GenCodec.md @@ -64,13 +64,51 @@ bound to any format. It only depends on the fact that this format is capable of * `Char`s, `String`s, `Boolean`s and `null`s * arbitrary byte chunks * millisecond-precision timestamps -* arbitrarily nested sequences (lists) -* arbitrarily nested objects, i.e. sequences of (string, value) pairs +* arbitrarily nested *lists* +* arbitrarily nested *objects*, i.e. sequences of *fields* - key-value mappings where each key is a string -Of course, if some type is not "natively" supported by some serialization format, it can be supported by representing -it with one of the primitive types. For example, timestamps may be serialized simply as `Long` values containing the +Of course, if some type is not "natively" supported by some serialization format, it can be supported by representing +it with one of the primitive types. For example, timestamps may be serialized simply as `Long` values containing the number of milliseconds since 01.01.1970. +### Writing *lists* and *objects* + +When a codec wants to write one of the simple values (e.g. `String` or `Int`) then it simply uses one of the direct methods on `Output`, e.g. `writeString` or `writeInt`. +But it may also write a *list* (an ordered sequence of values) or an *object* (*not-necessarily-ordered* sequence of *fields* - key-value mappings). Every list element or +object field value may be a list or object by itself, so serialized format may be arbitrarily nested, like JSON. You can think about this like a generalization +of JSON - similar logical structure (simple values, lists and objects) but without any particular syntax or representation enforced. + +If a codec wants to write a *list* then it must call `writeList` on an `Output` in order to obtain a +[`ListOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ListOutput.html) by calling `writeList` on `Output`. +Then it must call `writeElement` repetitively in order to obtain `Output` instances for each list element. Every element must be fully written before calling +`writeElement` again. After all elements have been written, the codec must call `finish()`. It's rather unlikely that you will ever need to do it +manually - usually you can just rely on macro generated codecs or use helpers like `GenCodec.createList`. + +If a codec wants to write an *object* then it must obtain an [`ObjectOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html) by +calling `writeObject`. `ObjectOutput` supports writing multiple named object fields. Then you can obtain `Output` instances for each object field value by calling +`writeField` (which takes field name as an argument). Just like with lists, a codec must fully write each field value before callilng `writeField` again and after +writing all fields, it must call `finish()`. Again, it's very unlikely that you'll have to implement this manually. Usually you can rely on macro generated codecs. +If you need a custom codec that writes an object, your best bet is probably to implement appropriate `apply` and `unapply` methods on companion object of your class. +And even if you can't do it (e.g. because you're writing a codec for a third party type) then you can still use +[`fromApplyUnapplyProvider`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec$.html#fromApplyUnapplyProvider[T](applyUnapplyProvider:Any):com.avsystem.commons.serialization.GenCodec[T]) +and avoid implementing your codec manually. + +#### Object field order + +For most serialization formats, it's completely natural to retain object field order. For example, a JSON string naturally has object fields stored in the order that they +were written to `JsonObjectOutput`. For these formats it is required that an `ObjectInput` returns object fields in exactly the same order as they were written to +a corresponding `ObjectOutput`. This normally includes all serialization formats backed by strings, byte sequences, streams, etc. + +However, there are also serialization formats that use memory representation where an object is usually backed by a hashtable. Such representations cannot retain field order. +`GenCodec` can still work with these but as an alternative to preserving field order, they must implement random field access (field-by-name access). This is done by +overriding [`peekField`](avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ObjectInput.html#peekField(name:String):com.avsystem.commons.misc.Opt[com.avsystem.commons.serialization.FieldInput]) on `ObjectInput`. + +To summarize, an `ObjectInput` is generally **not** guaranteed to retain order of fields as they were written to an `ObjectOutput` but if it doesn't then it must +provide random field access by field name. Also, note that out of all the default and macro-generated codecs provided, only [flat sealed hierarchy codecs](#flat-format) actually depend +on this requirement. All the other (non-custom) codecs ignore field order during deserialization. + +### Implementations of `Input` and `Output` available by default + The commons library contains example implementation of [`Input`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Input.html) and [`Output`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html) - [`SimpleValueInput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/SimpleValueInput.html) @@ -361,7 +399,9 @@ Similarly to case classes, sealed hierarchy codecs may be customized with annota * When using flat format, one of the case classes may be annotated as [`@defaultCase`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/defaultCase.html). When marker field is missing during deserialization, the codec will assume that it's deserializing the case class annotated as `@defaultCase`. This mechanism is useful to retain backwards compatibility when refactoring a case class into a sealed hierarchy with multiple case classes. -* It's important to remember that deserialization of the flat format relies on the preservation of field order by serialization backend. In particular, the marker field must come first so that the codec knows which class to create and how to deserialize the rest of the fields. However, there is one escape hatch from this requirement - a field present in one or more of case classes in the sealed hierarchy may be marked as [`@outOfOrder`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/outOfOrder.html). See the documentation of this annotation for more details. The direct motivation for introducing this annotation was to support deserialization of `_id` field in MongoDB documents - the database server always serves documents with `_id` being the very first field. +* It's important to remember that deserialization of the flat format relies on the preservation of field order by serialization backend (or random field access): + In particular, the marker field must be known to the codec before it reads other fields so that it knows which class to create and how to deserialize the rest of the fields. + There is one escape hatch from this requirement - a field present in one or more of case classes in the sealed hierarchy may be marked as [`@outOfOrder`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/outOfOrder.html). See the documentation of this annotation for more details. The direct motivation for introducing this annotation was to support deserialization of `_id` field in MongoDB documents - the database server always serves documents with `_id` being the very first field. ### Nested vs flat format From 6de24776aa4aea9999b7a9a8d322cba586ee3a00 Mon Sep 17 00:00:00 2001 From: ghik Date: Tue, 3 Apr 2018 14:17:17 +0200 Subject: [PATCH 3/4] cosmetics and docs --- .../commons/serialization/GenCodec.scala | 11 ++++++ .../commons/serialization/macroCodecs.scala | 35 ++++++++++--------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/GenCodec.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/GenCodec.scala index 1f697f932..849ae8861 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/GenCodec.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/GenCodec.scala @@ -142,6 +142,12 @@ object GenCodec extends RecursiveAutoCodecs with TupleGenCodecs { def createNonNullList[T](readFun: ListInput => T, writeFun: (ListOutput, T) => Any) = createList(readFun, writeFun, allowNull = false) + /** + * Helper method to manually implement a `GenCodec` that writes an object. NOTE: in most cases the easiest way to + * have a custom object codec is to manually implement `apply` and `unapply`/`unapplySeq` methods in companion object + * of your type or use [[fromApplyUnapplyProvider]] if the type comes from a third party code and you can't + * modify its companion object. + */ def createObject[T](readFun: ObjectInput => T, writeFun: (ObjectOutput, T) => Any, allowNull: Boolean) = new ObjectCodec[T] { def nullable = allowNull @@ -214,6 +220,11 @@ object GenCodec extends RecursiveAutoCodecs with TupleGenCodecs { } } + /** + * Convenience base class for `GenCodec`s that serialize values as objects. + * NOTE: if you need to implement a custom `GenCodec` that writes an object, the best way to do it is to have + * manually implemented `apply` and `unapply` in companion object or by using [[GenCodec.fromApplyUnapplyProvider]]. + */ trait ObjectCodec[T] extends NullSafeCodec[T] { def readObject(input: ObjectInput): T def writeObject(output: ObjectOutput, value: T): Unit diff --git a/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala b/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala index 8a75742b8..446030494 100644 --- a/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala +++ b/commons-core/src/main/scala/com/avsystem/commons/serialization/macroCodecs.scala @@ -151,30 +151,31 @@ abstract class FlatSealedHierarchyCodec[T]( } } - def read(): T = if (input.hasNext) { - val fi = input.nextField() - if (fi.fieldName == caseFieldName) readCase(fi) - else if (!oooFields.tryReadField(fi)) { - if (caseDependentFieldNames.contains(fi.fieldName)) { - if (defaultCaseIdx != -1) { - val defaultCaseName = caseNames(defaultCaseIdx) - val wrappedInput = new DefaultCaseObjectInput(fi, input, defaultCaseName) - readFlatCase(defaultCaseName, oooFields, wrappedInput, caseDeps(defaultCaseIdx)) + def read(): T = + if (input.hasNext) { + val fi = input.nextField() + if (fi.fieldName == caseFieldName) readCase(fi) + else if (!oooFields.tryReadField(fi)) { + if (caseDependentFieldNames.contains(fi.fieldName)) { + if (defaultCaseIdx != -1) { + val defaultCaseName = caseNames(defaultCaseIdx) + val wrappedInput = new DefaultCaseObjectInput(fi, input, defaultCaseName) + readFlatCase(defaultCaseName, oooFields, wrappedInput, caseDeps(defaultCaseIdx)) + } else { + missingCase(fi.fieldName) + } } else { - missingCase(fi.fieldName) + fi.skip() + read() } } else { - fi.skip() read() } + } else if (defaultCaseIdx != -1) { + readFlatCase(caseNames(defaultCaseIdx), oooFields, input, caseDeps(defaultCaseIdx)) } else { - read() + missingCase } - } else if (defaultCaseIdx != -1) { - readFlatCase(caseNames(defaultCaseIdx), oooFields, input, caseDeps(defaultCaseIdx)) - } else { - missingCase - } input.peekField(caseFieldName) match { case Opt(fi) => readCase(fi) From 068054eeb3112cb619f7f40f06f8c5bfae5ff0c8 Mon Sep 17 00:00:00 2001 From: ghik Date: Tue, 3 Apr 2018 14:20:59 +0200 Subject: [PATCH 4/4] doctoc update --- docs/GenCodec.md | 50 +++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/GenCodec.md b/docs/GenCodec.md index 581ab599b..3e4ec79a4 100644 --- a/docs/GenCodec.md +++ b/docs/GenCodec.md @@ -9,30 +9,32 @@ serialization libraries like [Circe](https://circe.github.io/circe/) or [uPickle ## Table of Contents *generated with [DocToc](https://github.com/thlorenz/doctoc)* -- [`GenCodec`](#gencodec) - - [`GenCodec` typeclass](#gencodec-typeclass) - - [Codecs available by default](#codecs-available-by-default) - - [`GenKeyCodec`](#genkeycodec) - - [Serializing and deserializing examples](#serializing-and-deserializing-examples) - - [Making your own types serializable](#making-your-own-types-serializable) - - [Simple types](#simple-types) - - [Case classes](#case-classes) - - [Safe evolution and refactoring - summary](#safe-evolution-and-refactoring---summary) - - [Case class like types](#case-class-like-types) - - [Singletons](#singletons) - - [Sealed hierarchies](#sealed-hierarchies) - - [Nested format](#nested-format) - - [Flat format](#flat-format) - - [Customizing sealed hierarchy codecs](#customizing-sealed-hierarchy-codecs) - - [Nested vs flat format](#nested-vs-flat-format) - - [Third party classes](#third-party-classes) - - [Summary](#summary) - - [Codec dependencies](#codec-dependencies) - - [Types supported by automatic materialization](#types-supported-by-automatic-materialization) - - [Recursive types, generic types and GADTs (generalized algebraic data types)](#recursive-types-generic-types-and-gadts-generalized-algebraic-data-types) - - [Customizing annotations](#customizing-annotations) - - [Safely introducing changes to serialized classes (retaining backwards compatibility)](#safely-introducing-changes-to-serialized-classes-retaining-backwards-compatibility) - - [Performance](#performance) +- [`GenCodec` typeclass](#gencodec-typeclass) + - [Writing *lists* and *objects*](#writing-lists-and-objects) + - [Object field order](#object-field-order) + - [Implementations of `Input` and `Output` available by default](#implementations-of-input-and-output-available-by-default) +- [Codecs available by default](#codecs-available-by-default) +- [`GenKeyCodec`](#genkeycodec) +- [Serializing and deserializing examples](#serializing-and-deserializing-examples) +- [Making your own types serializable](#making-your-own-types-serializable) +- [Simple types](#simple-types) +- [Case classes](#case-classes) + - [Safe evolution and refactoring - summary](#safe-evolution-and-refactoring---summary) + - [Case class like types](#case-class-like-types) +- [Singletons](#singletons) +- [Sealed hierarchies](#sealed-hierarchies) + - [Nested format](#nested-format) + - [Flat format](#flat-format) + - [Customizing sealed hierarchy codecs](#customizing-sealed-hierarchy-codecs) + - [Nested vs flat format](#nested-vs-flat-format) +- [Third party classes](#third-party-classes) +- [Summary](#summary) + - [Codec dependencies](#codec-dependencies) + - [Types supported by automatic materialization](#types-supported-by-automatic-materialization) + - [Recursive types, generic types and GADTs (generalized algebraic data types)](#recursive-types-generic-types-and-gadts-generalized-algebraic-data-types) + - [Customizing annotations](#customizing-annotations) + - [Safely introducing changes to serialized classes (retaining backwards compatibility)](#safely-introducing-changes-to-serialized-classes-retaining-backwards-compatibility) +- [Performance](#performance)