From b6932e1f3f47b0b5562972a93b1e0b9dd1b2ace7 Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Wed, 10 Apr 2019 23:41:19 +0800 Subject: [PATCH 1/6] SPARK-27416:UnsafeMapData & UnsafeArrayData Kryo serialization breaks when two machines have different Oops size --- .../catalyst/expressions/UnsafeArrayData.java | 35 ++++++---- .../catalyst/expressions/UnsafeDataUtils.java | 37 +++++++++++ .../catalyst/expressions/UnsafeMapData.java | 48 +++++++++++++- .../sql/catalyst/expressions/UnsafeRow.java | 9 +-- .../sql/catalyst/util/UnsafeArraySuite.scala | 27 +++++--- .../sql/catalyst/util/UnsafeMapSuite.scala | 64 +++++++++++++++++++ 6 files changed, 190 insertions(+), 30 deletions(-) create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java index 4ff0838ac117..1d215acafa86 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java @@ -25,6 +25,10 @@ import java.math.BigInteger; import java.nio.ByteBuffer; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; import org.apache.spark.sql.catalyst.util.ArrayData; import org.apache.spark.sql.types.*; import org.apache.spark.unsafe.Platform; @@ -58,7 +62,7 @@ * Instances of `UnsafeArrayData` act as pointers to row data stored in this format. */ -public final class UnsafeArrayData extends ArrayData implements Externalizable { +public final class UnsafeArrayData extends ArrayData implements Externalizable, KryoSerializable { public static int calculateHeaderPortionInBytes(int numFields) { return (int)calculateHeaderPortionInBytes((long)numFields); } @@ -492,17 +496,8 @@ public static UnsafeArrayData fromPrimitiveArray(double[] arr) { return fromPrimitiveArray(arr, Platform.DOUBLE_ARRAY_OFFSET, arr.length, 8); } - public byte[] getBytes() { - if (baseObject instanceof byte[] - && baseOffset == Platform.BYTE_ARRAY_OFFSET - && (((byte[]) baseObject).length == sizeInBytes)) { - return (byte[]) baseObject; - } else { - byte[] bytes = new byte[sizeInBytes]; - Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); - return bytes; - } + return UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); } @Override @@ -522,4 +517,22 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept this.baseObject = new byte[sizeInBytes]; in.readFully((byte[]) baseObject); } + + @Override + public void write(Kryo kryo, Output output) { + byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); + output.writeInt(bytes.length); + output.writeInt(this.numElements); + output.write(bytes); + } + + @Override + public void read(Kryo kryo, Input input) { + this.baseOffset = BYTE_ARRAY_OFFSET; + this.sizeInBytes = input.readInt(); + this.numElements = input.readInt(); + this.elementOffset = baseOffset + calculateHeaderPortionInBytes(this.numElements); + this.baseObject = new byte[sizeInBytes]; + input.read((byte[]) baseObject); + } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java new file mode 100644 index 000000000000..a0c0bf5bdeb3 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.expressions; + +import org.apache.spark.unsafe.Platform; + +/** + * General utilities available for unsafe data + */ +public class UnsafeDataUtils { + + public static byte[] getBytes(Object baseObject, long baseOffset, int sizeInBytes) { + if (baseObject instanceof byte[] + && baseOffset == Platform.BYTE_ARRAY_OFFSET + && (((byte[]) baseObject).length == sizeInBytes)) { + return (byte[]) baseObject; + } else { + byte[] bytes = new byte[sizeInBytes]; + Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); + return bytes; + } + } +} diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java index a0833a6df8bb..98a709f3076a 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeMapData.java @@ -17,11 +17,21 @@ package org.apache.spark.sql.catalyst.expressions; -import java.nio.ByteBuffer; - +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; import org.apache.spark.sql.catalyst.util.MapData; import org.apache.spark.unsafe.Platform; +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.nio.ByteBuffer; + +import static org.apache.spark.unsafe.Platform.BYTE_ARRAY_OFFSET; + /** * An Unsafe implementation of Map which is backed by raw memory instead of Java objects. * @@ -33,7 +43,7 @@ * elements, otherwise the behavior is undefined. */ // TODO: Use a more efficient format which doesn't depend on unsafe array. -public final class UnsafeMapData extends MapData { +public final class UnsafeMapData extends MapData implements Externalizable, KryoSerializable { private Object baseObject; private long baseOffset; @@ -123,4 +133,36 @@ public UnsafeMapData copy() { mapCopy.pointTo(mapDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); return mapCopy; } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); + out.writeInt(bytes.length); + out.write(bytes); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + this.baseOffset = BYTE_ARRAY_OFFSET; + this.sizeInBytes = in.readInt(); + this.baseObject = new byte[sizeInBytes]; + in.readFully((byte[]) baseObject); + pointTo(baseObject, baseOffset, sizeInBytes); + } + + @Override + public void write(Kryo kryo, Output output) { + byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); + output.writeInt(bytes.length); + output.write(bytes); + } + + @Override + public void read(Kryo kryo, Input input) { + this.baseOffset = BYTE_ARRAY_OFFSET; + this.sizeInBytes = input.readInt(); + this.baseObject = new byte[sizeInBytes]; + input.read((byte[]) baseObject); + pointTo(baseObject, baseOffset, sizeInBytes); + } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 11561fa7764a..8fd6029e976e 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -541,14 +541,7 @@ public boolean equals(Object other) { * Returns the underlying bytes for this UnsafeRow. */ public byte[] getBytes() { - if (baseObject instanceof byte[] && baseOffset == Platform.BYTE_ARRAY_OFFSET - && (((byte[]) baseObject).length == sizeInBytes)) { - return (byte[]) baseObject; - } else { - byte[] bytes = new byte[sizeInBytes]; - Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); - return bytes; - } + return UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); } // This is for debugging diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala index db25d2fb97e5..d426a5a00ea9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.util import java.time.ZoneId import org.apache.spark.{SparkConf, SparkFunSuite} -import org.apache.spark.serializer.JavaSerializer +import org.apache.spark.serializer.{KryoSerializer, JavaSerializer} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData @@ -60,6 +60,16 @@ class UnsafeArraySuite extends SparkFunSuite { val doubleMultiDimArray = Array( Array(1.1, 11.1), Array(2.2, 22.2, 222.2), Array(3.3, 33.3, 333.3, 3333.3)) + val serialArray = { + val offset = 32 + val data = new Array[Byte](1024) + Platform.putLong(data, offset, 1) + val arrayData = new UnsafeArrayData() + arrayData.pointTo(data, offset, data.length) + arrayData.setLong(0, 19285) + arrayData + } + test("read array") { val unsafeBoolean = ExpressionEncoder[Array[Boolean]].resolveAndBind(). toRow(booleanArray).getArray(0) @@ -214,14 +224,15 @@ class UnsafeArraySuite extends SparkFunSuite { } test("unsafe java serialization") { - val offset = 32 - val data = new Array[Byte](1024) - Platform.putLong(data, offset, 1) - val arrayData = new UnsafeArrayData() - arrayData.pointTo(data, offset, data.length) - arrayData.setLong(0, 19285) val ser = new JavaSerializer(new SparkConf).newInstance() - val arrayDataSer = ser.deserialize[UnsafeArrayData](ser.serialize(arrayData)) + val arrayDataSer = ser.deserialize[UnsafeArrayData](ser.serialize(serialArray)) + assert(arrayDataSer.getLong(0) == 19285) + assert(arrayDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024) + } + + test("unsafe Kryo serialization") { + val ser = new KryoSerializer(new SparkConf).newInstance() + val arrayDataSer = ser.deserialize[UnsafeArrayData](ser.serialize(serialArray)) assert(arrayDataSer.getLong(0) == 19285) assert(arrayDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala new file mode 100644 index 000000000000..735ecaa804fb --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.util + +import org.apache.spark.serializer.{JavaSerializer, KryoSerializer} +import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeMapData} +import org.apache.spark.unsafe.Platform +import org.apache.spark.{SparkConf, SparkFunSuite} + +class UnsafeMapSuite extends SparkFunSuite { + + val unsafeMapData = { + val offset = 32 + val keyArraySize = 256 + val baseObject = new Array[Byte](1024) + Platform.putLong(baseObject, offset, keyArraySize) + + val unsafeMap = new UnsafeMapData + Platform.putLong(baseObject, offset + 8, 1) + val keyArray = new UnsafeArrayData() + keyArray.pointTo(baseObject, offset + 8, keyArraySize) + keyArray.setLong(0, 19285) + + val valueArray = new UnsafeArrayData() + Platform.putLong(baseObject, offset + 8 + keyArray.getSizeInBytes, 1) + valueArray.pointTo(baseObject, offset + 8 + keyArray.getSizeInBytes, keyArraySize) + valueArray.setLong(0, 19285) + unsafeMap.pointTo(baseObject, offset, baseObject.length) + unsafeMap + } + + test("unsafe java serialization") { + val ser = new JavaSerializer(new SparkConf).newInstance() + val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData)) + assert(mapDataSer.numElements() == 1) + assert(mapDataSer.keyArray().getInt(0) == 19285) + assert(mapDataSer.valueArray().getInt(0) == 19285) + assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024) + } + + test("unsafe Kryo serialization") { + val ser = new KryoSerializer(new SparkConf).newInstance() + val mapDataSer = ser.deserialize[UnsafeMapData](ser.serialize(unsafeMapData)) + assert(mapDataSer.numElements() == 1) + assert(mapDataSer.keyArray().getInt(0) == 19285) + assert(mapDataSer.valueArray().getInt(0) == 19285) + assert(mapDataSer.getBaseObject.asInstanceOf[Array[Byte]].length == 1024) + } +} From 3876b8e978caf4dcb9e2f77a86e7acca07e052e3 Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Wed, 10 Apr 2019 23:56:43 +0800 Subject: [PATCH 2/6] refine code by remove useless method getBytes --- .../spark/sql/catalyst/expressions/UnsafeArrayData.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java index 1d215acafa86..18beae1cb983 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java @@ -496,13 +496,9 @@ public static UnsafeArrayData fromPrimitiveArray(double[] arr) { return fromPrimitiveArray(arr, Platform.DOUBLE_ARRAY_OFFSET, arr.length, 8); } - public byte[] getBytes() { - return UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); - } - @Override public void writeExternal(ObjectOutput out) throws IOException { - byte[] bytes = getBytes(); + byte[] bytes = UnsafeDataUtils.getBytes(baseObject, baseOffset, sizeInBytes); out.writeInt(bytes.length); out.writeInt(this.numElements); out.write(bytes); From 3a063487769fc6f15f1a47a3fd7cf11b3549776d Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Thu, 11 Apr 2019 11:11:17 +0800 Subject: [PATCH 3/6] java code style fixing --- .../org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala | 2 +- .../org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala index d426a5a00ea9..61ce63faf0d2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.util import java.time.ZoneId import org.apache.spark.{SparkConf, SparkFunSuite} -import org.apache.spark.serializer.{KryoSerializer, JavaSerializer} +import org.apache.spark.serializer.{JavaSerializer, KryoSerializer} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} import org.apache.spark.sql.catalyst.expressions.UnsafeArrayData diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala index 735ecaa804fb..a884cfe23075 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeMapSuite.scala @@ -17,10 +17,10 @@ package org.apache.spark.sql.catalyst.util +import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.serializer.{JavaSerializer, KryoSerializer} import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeMapData} import org.apache.spark.unsafe.Platform -import org.apache.spark.{SparkConf, SparkFunSuite} class UnsafeMapSuite extends SparkFunSuite { From 3a8990e091acef17d58b45774ef909da118eb139 Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Thu, 11 Apr 2019 12:37:54 +0800 Subject: [PATCH 4/6] fix java code style issue --- .../apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java index a0c0bf5bdeb3..ea171a61eea2 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java @@ -30,7 +30,8 @@ public static byte[] getBytes(Object baseObject, long baseOffset, int sizeInByte return (byte[]) baseObject; } else { byte[] bytes = new byte[sizeInBytes]; - Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); + Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, + sizeInBytes); return bytes; } } From 3a8ed0f8f173cd386fa835f5cb028ae76cdc9ebe Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Thu, 11 Apr 2019 20:44:07 +0800 Subject: [PATCH 5/6] remove useless else control --- .../spark/sql/catalyst/expressions/UnsafeDataUtils.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java index ea171a61eea2..35af994c83ff 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java @@ -28,11 +28,10 @@ public static byte[] getBytes(Object baseObject, long baseOffset, int sizeInByte && baseOffset == Platform.BYTE_ARRAY_OFFSET && (((byte[]) baseObject).length == sizeInBytes)) { return (byte[]) baseObject; - } else { - byte[] bytes = new byte[sizeInBytes]; - Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, - sizeInBytes); - return bytes; } + byte[] bytes = new byte[sizeInBytes]; + Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, + sizeInBytes); + return bytes; } } From fb566bcb6ce4d9cb601df776f493d2b5b870d360 Mon Sep 17 00:00:00 2001 From: mingbo_pb Date: Thu, 11 Apr 2019 22:50:55 +0800 Subject: [PATCH 6/6] make UnsafeDataUtils final & package private with private constructor for better control --- .../spark/sql/catalyst/expressions/UnsafeDataUtils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java index 35af994c83ff..74aa011abd63 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeDataUtils.java @@ -21,7 +21,10 @@ /** * General utilities available for unsafe data */ -public class UnsafeDataUtils { +final class UnsafeDataUtils { + + private UnsafeDataUtils() { + } public static byte[] getBytes(Object baseObject, long baseOffset, int sizeInBytes) { if (baseObject instanceof byte[]