diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java index 83a1b79f5749c..08c1a729995a0 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java @@ -110,7 +110,7 @@ public void setExcludes(List excludes) { * Allowed license families for this project. */ @Input - private List approvedLicenses = new ArrayList(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored")); + private List approvedLicenses = new ArrayList(Arrays.asList("SSPL+Elastic License", "Generated", "Vendored", "Apache LZ4-Java")); /** * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the @@ -154,6 +154,8 @@ public void runRat() { matchers.add(subStringMatcher("BSD4 ", "Original BSD License (with advertising clause)", "All advertising materials")); // Apache matchers.add(subStringMatcher("AL ", "Apache", "Licensed to Elasticsearch B.V. under one or more contributor")); + // Apache lz4-java + matchers.add(subStringMatcher("ALLZ4", "Apache LZ4-Java", "Copyright 2020 Adrien Grand and the lz4-java contributors")); // Generated resources matchers.add(subStringMatcher("GEN ", "Generated", "ANTLR GENERATED CODE")); // Vendored Code diff --git a/build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle b/build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle index 47a30ffc8ac40..9c6d3f8a8f291 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle @@ -55,6 +55,7 @@ def projectPathsToExclude = [ ':libs:elasticsearch-dissect', ':libs:elasticsearch-geo', ':libs:elasticsearch-grok', + ':libs:elasticsearch-lz4', ':libs:elasticsearch-nio', ':libs:elasticsearch-plugin-classloader', ':libs:elasticsearch-secure-sm', diff --git a/libs/lz4/build.gradle b/libs/lz4/build.gradle new file mode 100644 index 0000000000000..367da95ea6577 --- /dev/null +++ b/libs/lz4/build.gradle @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.publish' + +dependencies { + api 'org.lz4:lz4-java:1.8.0' + api project(':libs:elasticsearch-core') + + testImplementation(project(":test:framework")) { + exclude group: 'org.elasticsearch', module: 'elasticsearch-lz4' + } +} + +tasks.named("forbiddenPatterns").configure { + exclude '**/*.binary' +} + +tasks.named('forbiddenApisMain').configure { + // lz4 does not depend on core, so only jdk signatures should be checked + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // from java-lz4 + 'net.jpountz.util.UnsafeUtils' + ) +} diff --git a/server/licenses/lz4-java-1.8.0.jar.sha1 b/libs/lz4/licenses/lz4-java-1.8.0.jar.sha1 similarity index 100% rename from server/licenses/lz4-java-1.8.0.jar.sha1 rename to libs/lz4/licenses/lz4-java-1.8.0.jar.sha1 diff --git a/server/licenses/lz4-java-LICENSE.txt b/libs/lz4/licenses/lz4-java-LICENSE.txt similarity index 100% rename from server/licenses/lz4-java-LICENSE.txt rename to libs/lz4/licenses/lz4-java-LICENSE.txt diff --git a/server/licenses/lz4-java-NOTICE.txt b/libs/lz4/licenses/lz4-java-NOTICE.txt similarity index 100% rename from server/licenses/lz4-java-NOTICE.txt rename to libs/lz4/licenses/lz4-java-NOTICE.txt diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Compressor.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Compressor.java new file mode 100644 index 0000000000000..bdb06fa3b7ae7 --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Compressor.java @@ -0,0 +1,249 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Exception; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4JavaSafeCompressor. + * + * It modifies the original implementation to use custom LZ4SafeUtils and SafeUtils implementations which + * include performance improvements. Additionally, instead of allocating a new hashtable for each compress + * call, it reuses thread-local hashtables. Comments are included to mark the changes. + */ +public class ESLZ4Compressor extends LZ4Compressor { + + // Modified to add thread-local hash tables + private static final ThreadLocal sixtyFourKBHashTable = ThreadLocal.withInitial(() -> new short[8192]); + private static final ThreadLocal biggerHashTable = ThreadLocal.withInitial(() -> new int[4096]); + + public static final LZ4Compressor INSTANCE = new ESLZ4Compressor(); + + ESLZ4Compressor() { + } + + static int compress64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int destEnd) { + int srcEnd = srcOff + srcLen; + int srcLimit = srcEnd - 5; + int mflimit = srcEnd - 12; + int dOff = destOff; + int anchor = srcOff; + if (srcLen >= 13) { + // Modified to use thread-local hash table + short[] hashTable = sixtyFourKBHashTable.get(); + Arrays.fill(hashTable, (short) 0); + int sOff = srcOff + 1; + + label53: + while(true) { + int forwardOff = sOff; + int step = 1; + int var16 = 1 << LZ4Constants.SKIP_STRENGTH; + + int ref; + int excess; + do { + sOff = forwardOff; + forwardOff += step; + step = var16++ >>> LZ4Constants.SKIP_STRENGTH; + if (forwardOff > mflimit) { + break label53; + } + + excess = LZ4Utils.hash64k(SafeUtils.readInt(src, sOff)); + ref = srcOff + SafeUtils.readShort(hashTable, excess); + SafeUtils.writeShort(hashTable, excess, sOff - srcOff); + // Modified to use explicit == false + } while(LZ4SafeUtils.readIntEquals(src, ref, sOff) == false); + + excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor); + sOff -= excess; + ref -= excess; + int runLen = sOff - anchor; + int tokenOff = dOff++; + if (dOff + runLen + 8 + (runLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + + if (runLen >= 15) { + SafeUtils.writeByte(dest, tokenOff, 240); + dOff = LZ4SafeUtils.writeLen(runLen - 15, dest, dOff); + } else { + SafeUtils.writeByte(dest, tokenOff, runLen << 4); + } + + LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen); + dOff += runLen; + + while(true) { + SafeUtils.writeShortLE(dest, dOff, (short)(sOff - ref)); + dOff += 2; + sOff += 4; + ref += 4; + int matchLen = LZ4SafeUtils.commonBytes(src, ref, sOff, srcLimit); + if (dOff + 6 + (matchLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + + sOff += matchLen; + if (matchLen >= 15) { + SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | 15); + dOff = LZ4SafeUtils.writeLen(matchLen - 15, dest, dOff); + } else { + SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen); + } + + if (sOff > mflimit) { + anchor = sOff; + break label53; + } + + SafeUtils.writeShort(hashTable, LZ4Utils.hash64k(SafeUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff); + int h = LZ4Utils.hash64k(SafeUtils.readInt(src, sOff)); + ref = srcOff + SafeUtils.readShort(hashTable, h); + SafeUtils.writeShort(hashTable, h, sOff - srcOff); + // Modified to use explicit == false + if (LZ4SafeUtils.readIntEquals(src, sOff, ref) == false) { + anchor = sOff++; + break; + } + + tokenOff = dOff++; + SafeUtils.writeByte(dest, tokenOff, 0); + } + } + } + + dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd); + return dOff - destOff; + } + + public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { + SafeUtils.checkRange(src, srcOff, srcLen); + SafeUtils.checkRange(dest, destOff, maxDestLen); + int destEnd = destOff + maxDestLen; + if (srcLen < 65547) { + return compress64k(src, srcOff, srcLen, dest, destOff, destEnd); + } else { + int srcEnd = srcOff + srcLen; + int srcLimit = srcEnd - 5; + int mflimit = srcEnd - 12; + int dOff = destOff; + int sOff = srcOff + 1; + int anchor = srcOff; + // Modified to use thread-local hash table + int[] hashTable = biggerHashTable.get(); + Arrays.fill(hashTable, srcOff); + + label63: + while(true) { + int forwardOff = sOff; + int step = 1; + int var18 = 1 << LZ4Constants.SKIP_STRENGTH; + + while(true) { + sOff = forwardOff; + forwardOff += step; + step = var18++ >>> LZ4Constants.SKIP_STRENGTH; + if (forwardOff <= mflimit) { + int excess = LZ4Utils.hash(SafeUtils.readInt(src, sOff)); + int ref = SafeUtils.readInt(hashTable, excess); + int back = sOff - ref; + SafeUtils.writeInt(hashTable, excess, sOff); + // Modified to use explicit == false + if (back >= 65536 || LZ4SafeUtils.readIntEquals(src, ref, sOff) == false) { + continue; + } + + excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor); + sOff -= excess; + ref -= excess; + int runLen = sOff - anchor; + int tokenOff = dOff++; + if (dOff + runLen + 8 + (runLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + + if (runLen >= 15) { + SafeUtils.writeByte(dest, tokenOff, 240); + dOff = LZ4SafeUtils.writeLen(runLen - 15, dest, dOff); + } else { + SafeUtils.writeByte(dest, tokenOff, runLen << 4); + } + + LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen); + dOff += runLen; + + while(true) { + SafeUtils.writeShortLE(dest, dOff, back); + dOff += 2; + sOff += 4; + int matchLen = LZ4SafeUtils.commonBytes(src, ref + 4, sOff, srcLimit); + if (dOff + 6 + (matchLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + + sOff += matchLen; + if (matchLen >= 15) { + SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | 15); + dOff = LZ4SafeUtils.writeLen(matchLen - 15, dest, dOff); + } else { + SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen); + } + + if (sOff > mflimit) { + anchor = sOff; + break; + } + + SafeUtils.writeInt(hashTable, LZ4Utils.hash(SafeUtils.readInt(src, sOff - 2)), sOff - 2); + int h = LZ4Utils.hash(SafeUtils.readInt(src, sOff)); + ref = SafeUtils.readInt(hashTable, h); + SafeUtils.writeInt(hashTable, h, sOff); + back = sOff - ref; + // Modified to use explicit == false + if (back >= 65536 || LZ4SafeUtils.readIntEquals(src, ref, sOff) == false) { + anchor = sOff++; + continue label63; + } + + tokenOff = dOff++; + SafeUtils.writeByte(dest, tokenOff, 0); + } + } + + dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd); + return dOff - destOff; + } + } + } + } + + @Override + public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { + if (src.hasArray() && dest.hasArray()) { + return this.compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen); + } else { + throw new AssertionError("Do not support compression on direct buffers"); + } + } +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java new file mode 100644 index 0000000000000..ef887a9c5ae05 --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Exception; +import net.jpountz.lz4.LZ4FastDecompressor; + +import java.nio.ByteBuffer; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4JavaSafeFastDecompressor. + * + * It modifies the original implementation to use custom LZ4SafeUtils and SafeUtils implementations which + * include performance improvements. + */ +public class ESLZ4Decompressor extends LZ4FastDecompressor { + public static final LZ4FastDecompressor INSTANCE = new ESLZ4Decompressor(); + + ESLZ4Decompressor() { + } + + public int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) { + SafeUtils.checkRange(src, srcOff); + SafeUtils.checkRange(dest, destOff, destLen); + if (destLen == 0) { + if (SafeUtils.readByte(src, srcOff) != 0) { + throw new LZ4Exception("Malformed input at " + srcOff); + } else { + return 1; + } + } else { + int destEnd = destOff + destLen; + int sOff = srcOff; + int dOff = destOff; + + while(true) { + int token = SafeUtils.readByte(src, sOff) & 255; + ++sOff; + int literalLen = token >>> 4; + if (literalLen == 15) { + byte len; + for(boolean var11 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; literalLen += 255) { + } + + literalLen += len & 255; + } + + int literalCopyEnd = dOff + literalLen; + if (literalCopyEnd > destEnd - 8) { + if (literalCopyEnd != destEnd) { + throw new LZ4Exception("Malformed input at " + sOff); + } else { + LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen); + sOff += literalLen; + return sOff - srcOff; + } + } + + LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen); + sOff += literalLen; + int matchDec = SafeUtils.readShortLE(src, sOff); + sOff += 2; + int matchOff = literalCopyEnd - matchDec; + if (matchOff < destOff) { + throw new LZ4Exception("Malformed input at " + sOff); + } + + int matchLen = token & 15; + if (matchLen == 15) { + byte len; + for(boolean var15 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; matchLen += 255) { + } + + matchLen += len & 255; + } + + matchLen += 4; + int matchCopyEnd = literalCopyEnd + matchLen; + if (matchCopyEnd > destEnd - 8) { + if (matchCopyEnd > destEnd) { + throw new LZ4Exception("Malformed input at " + sOff); + } + + LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, literalCopyEnd, matchLen); + } else { + LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, literalCopyEnd, matchCopyEnd); + } + + dOff = matchCopyEnd; + } + } + } + + public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen) { + if (src.hasArray() && dest.hasArray()) { + return this.decompress(src.array(), srcOff + src.arrayOffset(), dest.array(), destOff + dest.arrayOffset(), destLen); + } else { + throw new AssertionError("Do not support decompression on direct buffers"); + } + } +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Constants.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Constants.java new file mode 100644 index 0000000000000..b1b7713472de8 --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Constants.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.elasticsearch.lz4; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4Constants. + * + * There are no modifications. It is copied to this package for reuse as the original implementation is + * package private. + */ +enum LZ4Constants { + ; + + static final int DEFAULT_COMPRESSION_LEVEL = 8+1; + static final int MAX_COMPRESSION_LEVEL = 16+1; + + static final int MEMORY_USAGE = 14; + static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6; + + static final int MIN_MATCH = 4; + + static final int HASH_LOG = MEMORY_USAGE - 2; + static final int HASH_TABLE_SIZE = 1 << HASH_LOG; + + static final int SKIP_STRENGTH = Math.max(NOT_COMPRESSIBLE_DETECTION_LEVEL, 2); + static final int COPY_LENGTH = 8; + static final int LAST_LITERALS = 5; + static final int MF_LIMIT = COPY_LENGTH + MIN_MATCH; + static final int MIN_LENGTH = MF_LIMIT + 1; + + static final int MAX_DISTANCE = 1 << 16; + + static final int ML_BITS = 4; + static final int ML_MASK = (1 << ML_BITS) - 1; + static final int RUN_BITS = 8 - ML_BITS; + static final int RUN_MASK = (1 << RUN_BITS) - 1; + + static final int LZ4_64K_LIMIT = (1 << 16) + (MF_LIMIT - 1); + static final int HASH_LOG_64K = HASH_LOG + 1; + static final int HASH_TABLE_SIZE_64K = 1 << HASH_LOG_64K; + + static final int HASH_LOG_HC = 15; + static final int HASH_TABLE_SIZE_HC = 1 << HASH_LOG_HC; + static final int OPTIMAL_ML = ML_MASK - 1 + MIN_MATCH; +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java new file mode 100644 index 0000000000000..32f357690560d --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java @@ -0,0 +1,479 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Exception; +import net.jpountz.util.Utils; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; + +import static org.elasticsearch.lz4.LZ4Constants.LAST_LITERALS; +import static org.elasticsearch.lz4.LZ4Constants.ML_BITS; +import static org.elasticsearch.lz4.LZ4Constants.ML_MASK; +import static org.elasticsearch.lz4.LZ4Constants.RUN_MASK; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4SafeUtils. + * + * It modifies the original implementation to use Java9 array mismatch method and varhandle performance + * improvements. Comments are included to mark the changes. + */ +enum LZ4SafeUtils { + ; + + // Added MethodHandles and static initialization + private static final MethodHandle readIntPlatformNative; + private static final MethodHandle writeIntPlatformNative; + private static final MethodHandle readLongPlatformNative; + private static final MethodHandle writeLongPlatformNative; + private static final MethodHandle copy4Bytes; + private static final MethodHandle copy8Bytes; + private static final MethodHandle commonBytes; + + static { + final MethodHandles.Lookup lookup = MethodHandles.lookup(); + + boolean exceptionCaught = false; + MethodHandle tempByteArrayViewVarHandle = null; + MethodHandle tempToMethodHandle = null; + Class accessModeClass = null; + try { + ClassLoader classLoader = AccessController.doPrivileged((PrivilegedAction) + () -> lookup.lookupClass().getClassLoader()); + Class varHandleClass = Class.forName("java.lang.invoke.VarHandle", true, classLoader); + accessModeClass = Class.forName("java.lang.invoke.VarHandle$AccessMode", true, classLoader); + MethodType t = MethodType.methodType(varHandleClass, Class.class, ByteOrder.class); + tempByteArrayViewVarHandle = AccessController.doPrivileged((PrivilegedExceptionAction) + () -> lookup.findStatic(MethodHandles.class, "byteArrayViewVarHandle", t)); + MethodType toMethodHandleType = MethodType.methodType(MethodHandle.class, accessModeClass); + tempToMethodHandle = lookup.findVirtual(varHandleClass, "toMethodHandle", toMethodHandleType); + } catch (Exception ignored) { + exceptionCaught = true; + } + final MethodHandle byteArrayViewVarHandle = tempByteArrayViewVarHandle; + final MethodHandle toMethodHandle = tempToMethodHandle; + @SuppressWarnings({"unchecked", "rawtypes"}) + final Object getAccessModeEnum = accessModeClass != null ? Enum.valueOf((Class) accessModeClass, "GET") : null; + @SuppressWarnings({"unchecked", "rawtypes"}) + final Object setAccessModeEnum = accessModeClass != null ? Enum.valueOf((Class) accessModeClass, "SET") : null; + + boolean initialized = exceptionCaught == false && byteArrayViewVarHandle != null && toMethodHandle != null + && getAccessModeEnum != null && setAccessModeEnum != null; + + final Object intVarHandle; + final Object longVarHandle; + if (initialized) { + intVarHandle = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + return byteArrayViewVarHandle.invoke(int[].class, Utils.NATIVE_BYTE_ORDER); + } catch (Throwable ignored) {} + return null; + }); + longVarHandle = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + return byteArrayViewVarHandle.invoke(long[].class, Utils.NATIVE_BYTE_ORDER); + } catch (Throwable ignored) {} + return null; + }); + } else { + intVarHandle = null; + longVarHandle = null; + } + + readIntPlatformNative = AccessController.doPrivileged((PrivilegedAction) () -> { + if (intVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(intVarHandle, getAccessModeEnum); + } catch (Throwable ignored) {} + } + return null; + }); + + writeIntPlatformNative = AccessController.doPrivileged((PrivilegedAction) () -> { + if (intVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(intVarHandle, setAccessModeEnum); + } catch (Throwable ignored) {} + } + return null; + }); + + readLongPlatformNative = AccessController.doPrivileged((PrivilegedAction) () -> { + if (longVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(longVarHandle, getAccessModeEnum); + } catch (Throwable ignored) {} + + } + return null; + }); + + writeLongPlatformNative = AccessController.doPrivileged((PrivilegedAction) () -> { + if (longVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(longVarHandle, setAccessModeEnum); + } catch (Throwable ignored) {} + + } + return null; + }); + + copy4Bytes = AccessController.doPrivileged((PrivilegedAction) () -> { + if (readIntPlatformNative != null && writeIntPlatformNative != null) { + try { + final MethodType type = MethodType.methodType(void.class, byte[].class, int.class, byte[].class, int.class); + return lookup.findStatic(LZ4SafeUtils.class, "mhCopy4Bytes", type); + } catch (Throwable ignored) {} + } + try { + final MethodType type = MethodType.methodType(void.class, byte[].class, int.class, byte[].class, int.class); + return lookup.findStatic(LZ4SafeUtils.class, "legacyCopy4Bytes", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + + copy8Bytes = AccessController.doPrivileged((PrivilegedAction) () -> { + if (readLongPlatformNative != null && writeLongPlatformNative != null) { + try { + final MethodType type = MethodType.methodType(void.class, byte[].class, int.class, byte[].class, int.class); + return lookup.findStatic(LZ4SafeUtils.class, "mhCopy8Bytes", type); + } catch (Throwable ignored) {} + } + try { + final MethodType type = MethodType.methodType(void.class, byte[].class, int.class, byte[].class, int.class); + return lookup.findStatic(LZ4SafeUtils.class, "legacyCopy8Bytes", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + + commonBytes = AccessController.doPrivileged((PrivilegedAction) () -> { + MethodType type = MethodType.methodType(int.class, byte[].class, int.class, int.class, byte[].class, int.class, int.class); + if (initialized) { + try { + return lookup.findStatic(Arrays.class, "mismatch", type); + } catch (Throwable ignored) {} + } + try { + return lookup.findStatic(LZ4SafeUtils.class, "legacyCommonBytes", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + } + + static int hash(byte[] buf, int i) { + return LZ4Utils.hash(SafeUtils.readInt(buf, i)); + } + + static int hash64k(byte[] buf, int i) { + return LZ4Utils.hash64k(SafeUtils.readInt(buf, i)); + } + + static boolean readIntEquals(byte[] buf, int i, int j) { + return SafeUtils.readInt(buf, i) == SafeUtils.readInt(buf, j); + } + + static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) { + for (int i = 0; i < matchLen; ++i) { + dest[dOff + i] = dest[matchOff + i]; + } + } + + // Modified wildIncrementalCopy to mirror version in LZ4UnsafeUtils + static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) { + if (dOff - matchOff < 4) { + for (int i = 0; i < 4; ++i) { + dest[dOff + i] = dest[matchOff + i]; + } + dOff += 4; + matchOff += 4; + int dec = 0; + assert dOff >= matchOff && dOff - matchOff < 8; + switch (dOff - matchOff) { + case 1: + matchOff -= 3; + break; + case 2: + matchOff -= 2; + break; + case 3: + matchOff -= 3; + dec = -1; + break; + case 5: + dec = 1; + break; + case 6: + dec = 2; + break; + case 7: + dec = 3; + break; + default: + break; + } + + copy4Bytes(dest, matchOff, dest, dOff); + dOff += 4; + matchOff -= dec; + } else if (dOff - matchOff < LZ4Constants.COPY_LENGTH) { + copy8Bytes(dest, matchOff, dest, dOff); + dOff += dOff - matchOff; + } + while (dOff < matchCopyEnd) { + copy8Bytes(dest, matchOff, dest, dOff); + dOff += 8; + matchOff += 8; + } + } + + // Added legacy method to copy 8 bytes incrementally + private static void legacyCopy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + for (int i = 0; i < 8; ++i) { + dest[dOff + i] = src[sOff + i]; + } + } + + // Added to read long. Having a dedicated private static method, makes the JDK optimization consistent. + private static long readLong(byte[] bytes, int off) { + try { + return (long) readLongPlatformNative.invokeExact(bytes, off); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Added to write long. Having a dedicated private static method, makes the JDK optimization consistent. + private static void writeLong(byte[] bytes, int off, long value) { + try { + writeLongPlatformNative.invokeExact(bytes, off, value); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Added to call MethodHandles + private static void mhCopy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + writeLong(dest, dOff, readLong(src, sOff)); + } + + // Modified to use MethodHandle + static void copy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + try { + copy8Bytes.invokeExact(src, sOff, dest, dOff); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Added legacy method to copy 4 bytes incrementally + private static void legacyCopy4Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + for (int i = 0; i < 4; ++i) { + dest[dOff + i] = src[sOff + i]; + } + } + + // Added to read int. Having a dedicated private static method, makes the JDK optimization consistent. + private static int readInt(byte[] bytes, int off) { + try { + return (int) readIntPlatformNative.invokeExact(bytes, off); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Added to write int. Having a dedicated private static method, makes the JDK optimization consistent. + private static void writeInt(byte[] bytes, int off, int value) { + try { + writeIntPlatformNative.invokeExact(bytes, off, value); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Added to call MethodHandles + private static void mhCopy4Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + writeInt(dest, dOff, readInt(src, sOff)); + } + + // Added to call method handle + static void copy4Bytes(byte[] src, int sOff, byte[] dest, int dOff) { + try { + copy4Bytes.invokeExact(src, sOff, dest, dOff); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + private static int legacyCommonBytes(byte[] b, int o1, int ignored1, byte[] ignored2, int o2, int limit) { + int count = 0; + while (o2 < limit && b[o1++] == b[o2++]) { + ++count; + } + return count; + } + + // Modified to use Arrays.mismatch + static int commonBytes(byte[] b, int o1, int o2, int limit) { + try { + int mismatch = (int) commonBytes.invokeExact(b, o1, limit, b, o2, limit); + return mismatch == -1 ? limit : mismatch; + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) { + int count = 0; + while (o1 > l1 && o2 > l2 && b[--o1] == b[--o2]) { + ++count; + } + return count; + } + + static void safeArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) { + System.arraycopy(src, sOff, dest, dOff, len); + } + + static void wildArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) { + try { + for (int i = 0; i < len; i += 8) { + copy8Bytes(src, sOff + i, dest, dOff + i); + } + // Modified to catch IndexOutOfBoundsException instead of ArrayIndexOutOfBoundsException. + // VarHandles throw IndexOutOfBoundsException + } catch (IndexOutOfBoundsException e) { + throw new LZ4Exception("Malformed input at offset " + sOff); + } + } + + static int encodeSequence(byte[] src, int anchor, int matchOff, int matchRef, int matchLen, byte[] dest, int dOff, int destEnd) { + final int runLen = matchOff - anchor; + final int tokenOff = dOff++; + + if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + + int token; + if (runLen >= RUN_MASK) { + token = (byte) (RUN_MASK << ML_BITS); + dOff = writeLen(runLen - RUN_MASK, dest, dOff); + } else { + token = runLen << ML_BITS; + } + + // copy literals + wildArraycopy(src, anchor, dest, dOff, runLen); + dOff += runLen; + + // encode offset + final int matchDec = matchOff - matchRef; + dest[dOff++] = (byte) matchDec; + dest[dOff++] = (byte) (matchDec >>> 8); + + // encode match len + matchLen -= 4; + if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) { + throw new LZ4Exception("maxDestLen is too small"); + } + if (matchLen >= ML_MASK) { + token |= ML_MASK; + dOff = writeLen(matchLen - RUN_MASK, dest, dOff); + } else { + token |= matchLen; + } + + dest[tokenOff] = (byte) token; + + return dOff; + } + + static int lastLiterals(byte[] src, int sOff, int srcLen, byte[] dest, int dOff, int destEnd) { + final int runLen = srcLen; + + if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) { + throw new LZ4Exception(); + } + + if (runLen >= RUN_MASK) { + dest[dOff++] = (byte) (RUN_MASK << ML_BITS); + dOff = writeLen(runLen - RUN_MASK, dest, dOff); + } else { + dest[dOff++] = (byte) (runLen << ML_BITS); + } + // copy literals + System.arraycopy(src, sOff, dest, dOff, runLen); + dOff += runLen; + + return dOff; + } + + static int writeLen(int len, byte[] dest, int dOff) { + while (len >= 0xFF) { + dest[dOff++] = (byte) 0xFF; + len -= 0xFF; + } + dest[dOff++] = (byte) len; + return dOff; + } + + static class Match { + int start, ref, len; + + void fix(int correction) { + start += correction; + ref += correction; + len -= correction; + } + + int end() { + return start + len; + } + } + + static void copyTo(LZ4SafeUtils.Match m1, LZ4SafeUtils.Match m2) { + m2.len = m1.len; + m2.start = m1.start; + m2.ref = m1.ref; + } +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java new file mode 100644 index 0000000000000..496148ce2196c --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.elasticsearch.lz4; + +import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG; +import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG_64K; +import static org.elasticsearch.lz4.LZ4Constants.HASH_LOG_HC; +import static org.elasticsearch.lz4.LZ4Constants.MIN_MATCH; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4Utils. + * + * There are no modifications. It is copied to this package for reuse as the original implementation is + * package private. + */ +enum LZ4Utils { + ; + + private static final int MAX_INPUT_SIZE = 0x7E000000; + + static int maxCompressedLength(int length) { + if (length < 0) { + throw new IllegalArgumentException("length must be >= 0, got " + length); + } else if (length >= MAX_INPUT_SIZE) { + throw new IllegalArgumentException("length must be < " + MAX_INPUT_SIZE); + } + return length + length / 255 + 16; + } + + static int hash(int i) { + return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG); + } + + static int hash64k(int i) { + return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_64K); + } + + static int hashHC(int i) { + return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_HC); + } + + static class Match { + int start, ref, len; + + void fix(int correction) { + start += correction; + ref += correction; + len -= correction; + } + + int end() { + return start + len; + } + } + + static void copyTo(LZ4Utils.Match m1, LZ4Utils.Match m2) { + m2.len = m1.len; + m2.start = m1.start; + m2.ref = m1.ref; + } +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/SafeUtils.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/SafeUtils.java new file mode 100644 index 0000000000000..adc612406679b --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/SafeUtils.java @@ -0,0 +1,246 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.elasticsearch.lz4; + +import net.jpountz.util.Utils; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.security.PrivilegedExceptionAction; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.SafeUtils. + * + * It modifies the original implementation to use Java9 varhandle performance improvements. Comments + * are included to mark the changes. + */ +public enum SafeUtils { + ; + + // Added MethodHandles and static initialization + private static final MethodHandle readShortLittleEndian; + private static final MethodHandle writeShortLittleEndian; + private static final MethodHandle readIntPlatformNative; + + static { + final MethodHandles.Lookup lookup = MethodHandles.lookup(); + + boolean exceptionCaught = false; + MethodHandle tempByteArrayViewVarHandle = null; + MethodHandle tempToMethodHandle = null; + Class accessModeClass = null; + try { + ClassLoader classLoader = AccessController.doPrivileged((PrivilegedAction) + () -> lookup.lookupClass().getClassLoader()); + Class varHandleClass = Class.forName("java.lang.invoke.VarHandle", true, classLoader); + accessModeClass = Class.forName("java.lang.invoke.VarHandle$AccessMode", true, classLoader); + MethodType t = MethodType.methodType(varHandleClass, Class.class, ByteOrder.class); + tempByteArrayViewVarHandle = AccessController.doPrivileged((PrivilegedExceptionAction) + () -> lookup.findStatic(MethodHandles.class, "byteArrayViewVarHandle", t)); + MethodType toMethodHandleType = MethodType.methodType(MethodHandle.class, accessModeClass); + tempToMethodHandle = lookup.findVirtual(varHandleClass, "toMethodHandle", toMethodHandleType); + } catch (Exception ignored) { + exceptionCaught = true; + } + final MethodHandle byteArrayViewVarHandle = tempByteArrayViewVarHandle; + final MethodHandle toMethodHandle = tempToMethodHandle; + @SuppressWarnings({"unchecked", "rawtypes"}) + final Object getAccessModeEnum = accessModeClass != null ? Enum.valueOf((Class) accessModeClass, "GET") : null; + @SuppressWarnings({"unchecked", "rawtypes"}) + final Object setAccessModeEnum = accessModeClass != null ? Enum.valueOf((Class) accessModeClass, "SET") : null; + + boolean initialized = exceptionCaught == false && byteArrayViewVarHandle != null && toMethodHandle != null + && getAccessModeEnum != null && setAccessModeEnum != null; + + Object shortLEVarHandle; + Object intVarHandle; + if (initialized) { + shortLEVarHandle = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + return byteArrayViewVarHandle.invoke(short[].class, ByteOrder.LITTLE_ENDIAN); + } catch (Throwable ignored) {} + return null; + }); + intVarHandle = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + return byteArrayViewVarHandle.invoke(int[].class, Utils.NATIVE_BYTE_ORDER); + } catch (Throwable ignored) {} + return null; + }); + } else { + shortLEVarHandle = null; + intVarHandle = null; + } + + readShortLittleEndian = AccessController.doPrivileged((PrivilegedAction) () -> { + if (shortLEVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(shortLEVarHandle, getAccessModeEnum); + } catch (Throwable ignored) {} + } + try { + final MethodType type = MethodType.methodType(short.class, byte[].class, int.class); + return lookup.findStatic(SafeUtils.class, "legacyReadShortLE", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + + writeShortLittleEndian = AccessController.doPrivileged((PrivilegedAction) () -> { + if (shortLEVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(shortLEVarHandle, setAccessModeEnum); + } catch (Throwable ignored) {} + } + try { + final MethodType type = MethodType.methodType(void.class, byte[].class, int.class, short.class); + return lookup.findStatic(SafeUtils.class, "legacyWriteShortLE", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + + readIntPlatformNative = AccessController.doPrivileged((PrivilegedAction) () -> { + if (intVarHandle != null) { + try { + return (MethodHandle) toMethodHandle.invoke(intVarHandle, getAccessModeEnum); + } catch (Throwable ignored) {} + + } + try { + final MethodType type = MethodType.methodType(int.class, byte[].class, int.class); + return lookup.findStatic(SafeUtils.class, "legacyReadInt", type); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + } + + + public static void checkRange(byte[] buf, int off) { + if (off < 0 || off >= buf.length) { + throw new ArrayIndexOutOfBoundsException(off); + } + } + + public static void checkRange(byte[] buf, int off, int len) { + checkLength(len); + if (len > 0) { + checkRange(buf, off); + checkRange(buf, off + len - 1); + } + } + + public static void checkLength(int len) { + if (len < 0) { + throw new IllegalArgumentException("lengths must be >= 0"); + } + } + + public static byte readByte(byte[] buf, int i) { + return buf[i]; + } + + private static int readIntBE(byte[] buf, int i) { + return (buf[i] & 255) << 24 | (buf[i + 1] & 255) << 16 | (buf[i + 2] & 255) << 8 | buf[i + 3] & 255; + } + + private static int readIntLE(byte[] buf, int i) { + return buf[i] & 255 | (buf[i + 1] & 255) << 8 | (buf[i + 2] & 255) << 16 | (buf[i + 3] & 255) << 24; + } + + // Modified to rename to legacy read int method + private static int legacyReadInt(byte[] buf, int i) { + return Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN ? readIntBE(buf, i) : readIntLE(buf, i); + } + + // Modified to use writeShortLE + public static int readInt(byte[] buf, int i) { + try { + return (int) readIntPlatformNative.invokeExact(buf, i); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + // Unused in forked instance, no need to optimize + public static long readLongLE(byte[] buf, int i) { + return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24) + | ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56); + } + + // Modified to rename to legacy write short method + private static void legacyWriteShortLE(byte[] buf, int off, short v) { + buf[off++] = (byte) v; + buf[off++] = (byte) (v >>> 8); + } + + // Modified to use MethodHandle + public static void writeShortLE(byte[] buf, int off, int v) { + try { + writeShortLittleEndian.invokeExact(buf, off, (short) v); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + public static void writeInt(int[] buf, int off, int v) { + buf[off] = v; + } + + public static int readInt(int[] buf, int off) { + return buf[off]; + } + + public static void writeByte(byte[] dest, int off, int i) { + dest[off] = (byte) i; + } + + public static void writeShort(short[] buf, int off, int v) { + buf[off] = (short) v; + } + + // Modified to rename to legacy read short method + private static short legacyReadShortLE(byte[] buf, int i) { + return (short) (buf[i] & 255 | (buf[i + 1] & 255) << 8); + } + + // Modified to use MethodHandle + public static int readShortLE(byte[] buf, int i) { + try { + return Short.toUnsignedInt((short) readShortLittleEndian.invokeExact(buf, i)); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + public static int readShort(short[] buf, int off) { + return buf[off] & 0xFFFF; + } +} diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/package-info.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/package-info.java new file mode 100644 index 0000000000000..89c6e944c24ec --- /dev/null +++ b/libs/lz4/src/main/java/org/elasticsearch/lz4/package-info.java @@ -0,0 +1,20 @@ +/* @notice + * + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ + +package org.elasticsearch.lz4; diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/AbstractLZ4TestCase.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/AbstractLZ4TestCase.java new file mode 100644 index 0000000000000..98a5319506e99 --- /dev/null +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/AbstractLZ4TestCase.java @@ -0,0 +1,379 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ + +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4CompressorWithLength; +import net.jpountz.lz4.LZ4DecompressorWithLength; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4SafeDecompressor; + +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.AbstractLZ4Test. + * + * It modifies the abstract test case to only test byte arrays and byte array backed byte buffers. These are + * the only bytes we support. + */ +abstract class AbstractLZ4TestCase extends ESTestCase { + + public interface TesterBase { + + T allocate(int length); + T copyOf(byte[] array); + byte[] copyOf(T data, int off, int len); + int maxCompressedLength(int len); + void fill(T instance, byte b); + + // Modified to remove redundant modifiers + class ByteArrayTesterBase implements TesterBase { + + @Override + public byte[] allocate(int length) { + return new byte[length]; + } + + @Override + public byte[] copyOf(byte[] array) { + return Arrays.copyOf(array, array.length); + } + + @Override + public byte[] copyOf(byte[] data, int off, int len) { + return Arrays.copyOfRange(data, off, off + len); + } + + @Override + public int maxCompressedLength(int len) { + return LZ4Utils.maxCompressedLength(len); + } + + @Override + public void fill(byte[] instance, byte b) { + Arrays.fill(instance, b); + } + } + + // Modified to remove redundant modifiers + class ByteBufferTesterBase implements TesterBase { + + @Override + public ByteBuffer allocate(int length) { + ByteBuffer bb; + int slice = randomInt(5); + // Modified to only test heap ByteBuffers + bb = ByteBuffer.allocate(length + slice); + bb.position(slice); + bb = bb.slice(); + if (randomBoolean()) { + bb.order(ByteOrder.LITTLE_ENDIAN); + } else { + bb.order(ByteOrder.BIG_ENDIAN); + } + return bb; + } + + @Override + public ByteBuffer copyOf(byte[] array) { + ByteBuffer bb = allocate(array.length).put(array); + // Modified to not test read only buffers as they do not make the array accessible + bb.position(0); + return bb; + } + + @Override + public byte[] copyOf(ByteBuffer data, int off, int len) { + byte[] copy = new byte[len]; + data.position(off); + data.get(copy); + return copy; + } + + @Override + public int maxCompressedLength(int len) { + return LZ4Utils.maxCompressedLength(len); + } + + @Override + public void fill(ByteBuffer instance, byte b) { + for (int i = 0; i < instance.capacity(); ++i) { + instance.put(i, b); + } + } + } + } + + public interface Tester extends TesterBase { + + int compress(LZ4Compressor compressor, T src, int srcOff, int srcLen, T dest, int destOff, int maxDestLen); + int decompress(LZ4FastDecompressor decompressor, T src, int srcOff, T dest, int destOff, int destLen); + int decompress(LZ4SafeDecompressor decompressor, T src, int srcOff, int srcLen, T dest, int destOff, int maxDestLen); + + // Modified to remove redundant modifiers + class ByteArrayTester extends ByteArrayTesterBase implements Tester { + + @Override + public int compress(LZ4Compressor compressor, byte[] src, int srcOff, + int srcLen, byte[] dest, int destOff, int maxDestLen) { + return compressor.compress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, + byte[] src, int srcOff, byte[] dest, int destOff, int destLen) { + return decompressor.decompress(src, srcOff, dest, destOff, destLen); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, + byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { + return decompressor.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + } + // Modified to remove redundant modifiers + Tester BYTE_ARRAY = new ByteArrayTester(); + // Modified to remove redundant modifiers + Tester BYTE_ARRAY_WITH_LENGTH = new ByteArrayTester() { + @Override + public int compress(LZ4Compressor compressor, byte[] src, int srcOff, + int srcLen, byte[] dest, int destOff, int maxDestLen) { + return new LZ4CompressorWithLength(compressor).compress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, + byte[] src, int srcOff, byte[] dest, int destOff, int destLen) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, dest, destOff); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, + byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, srcLen, dest, destOff); + } + }; + + // Modified to remove redundant modifiers + class ByteBufferTester extends ByteBufferTesterBase implements Tester { + + @Override + public int compress(LZ4Compressor compressor, ByteBuffer src, int srcOff, + int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { + return compressor.compress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, + int srcOff, ByteBuffer dest, int destOff, int destLen) { + return decompressor.decompress(src, srcOff, dest, destOff, destLen); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, + int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { + return decompressor.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + } + // Modified to remove redundant modifiers + Tester BYTE_BUFFER = new ByteBufferTester(); + // Modified to remove redundant modifiers + Tester BYTE_BUFFER_WITH_LENGTH = new ByteBufferTester() { + @Override + public int compress(LZ4Compressor compressor, ByteBuffer src, int srcOff, + int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { + return new LZ4CompressorWithLength(compressor).compress(src, srcOff, srcLen, dest, destOff, maxDestLen); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, + int srcOff, ByteBuffer dest, int destOff, int destLen) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, dest, destOff); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, + int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, srcOff, srcLen, dest, destOff); + } + }; + } + + // Tester to test a simple compress/decompress(src, dest) type of APIs + public interface SrcDestTester extends TesterBase { + + int compress(LZ4Compressor compressor, T src, T dest); + int decompress(LZ4FastDecompressor decompressor, T src, T dest); + int decompress(LZ4SafeDecompressor decompressor, T src, T dest); + + // Modified to remove redundant modifiers + class ByteArrayTester extends ByteArrayTesterBase implements SrcDestTester { + + @Override + public int compress(LZ4Compressor compressor, byte[] src, byte[] dest) { + return compressor.compress(src, dest); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, byte[] src, byte[] dest) { + return decompressor.decompress(src, dest); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, byte[] src, byte[] dest) { + return decompressor.decompress(src, dest); + } + } + // Modified to remove redundant modifiers + SrcDestTester BYTE_ARRAY = new ByteArrayTester(); + // Modified to remove redundant modifiers + SrcDestTester BYTE_ARRAY_WITH_LENGTH = new ByteArrayTester() { + @Override + public int compress(LZ4Compressor compressor, byte[] src, byte[] dest) { + return new LZ4CompressorWithLength(compressor).compress(src, dest); + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, byte[] src, byte[] dest) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, dest); + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, byte[] src, byte[] dest) { + return new LZ4DecompressorWithLength(decompressor).decompress(src, dest); + } + }; + + // Modified to remove redundant modifiers + class ByteBufferTester extends ByteBufferTesterBase implements SrcDestTester { + + @Override + public int compress(LZ4Compressor compressor, ByteBuffer src, ByteBuffer dest) { + final int pos = dest.position(); + compressor.compress(src, dest); + return dest.position() - pos; + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, ByteBuffer dest) { + final int pos = src.position(); + decompressor.decompress(src, dest); + return src.position() - pos; + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, ByteBuffer dest) { + final int pos = dest.position(); + decompressor.decompress(src, dest); + return dest.position() - pos; + } + } + // Modified to remove redundant modifiers + SrcDestTester BYTE_BUFFER = new ByteBufferTester(); + // Modified to remove redundant modifiers + SrcDestTester BYTE_BUFFER_WITH_LENGTH = new ByteBufferTester() { + @Override + public int compress(LZ4Compressor compressor, ByteBuffer src, ByteBuffer dest) { + final int pos = dest.position(); + new LZ4CompressorWithLength(compressor).compress(src, dest); + return dest.position() - pos; + } + + @Override + public int decompress(LZ4FastDecompressor decompressor, ByteBuffer src, ByteBuffer dest) { + final int pos = src.position(); + new LZ4DecompressorWithLength(decompressor).decompress(src, dest); + return src.position() - pos; + } + + @Override + public int decompress(LZ4SafeDecompressor decompressor, ByteBuffer src, ByteBuffer dest) { + final int pos = dest.position(); + new LZ4DecompressorWithLength(decompressor).decompress(src, dest); + return dest.position() - pos; + } + }; + } + + protected class RandomBytes { + private final byte[] bytes; + RandomBytes(int n) { + assert n > 0 && n <= 256; + bytes = new byte[n]; + for (int i = 0; i < n; ++i) { + bytes[i] = (byte) randomInt(255); + } + } + byte next() { + final int i = randomInt(bytes.length - 1); + return bytes[i]; + } + } + + protected static byte[] readResource(String resource) throws IOException { + InputStream is = AbstractLZ4TestCase.class.getResourceAsStream(resource); + if (is == null) { + throw new IllegalStateException("Cannot find " + resource); + } + byte[] buf = new byte[4096]; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + while (true) { + final int read = is.read(buf); + if (read == -1) { + break; + } + baos.write(buf, 0, read); + } + } finally { + is.close(); + } + return baos.toByteArray(); + } + + protected byte[] randomArray(int len, int n) { + byte[] result = new byte[len]; + RandomBytes randomBytes = new RandomBytes(n); + for (int i = 0; i < result.length; ++i) { + result[i] = randomBytes.next(); + } + return result; + } + + protected ByteBuffer copyOf(byte[] bytes, int offset, int len) { + ByteBuffer buffer; + // Modified to only test heap ByteBuffers + buffer = ByteBuffer.allocate(bytes.length); + buffer.put(bytes); + buffer.position(offset); + buffer.limit(offset + len); + if (randomBoolean()) { + buffer = buffer.asReadOnlyBuffer(); + } + return buffer; + } +} diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java new file mode 100644 index 0000000000000..37e97b49452f2 --- /dev/null +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4CompressorTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Compressor; + +import net.jpountz.lz4.LZ4Factory; + +import net.jpountz.lz4.LZ4FastDecompressor; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class ESLZ4CompressorTests extends ESTestCase { + + public void testCompressRealisticUnicode() { + for (int i = 0; i < 15; ++i) { + int stringLengthMultiplier = randomFrom(5, 10, 20, 40, 80, 160, 320); + + final String uncompressedString = randomRealisticUnicodeOfCodepointLength(stringLengthMultiplier * 1024); + byte[] uncompressed = uncompressedString.getBytes(StandardCharsets.UTF_8); + + byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + byte[] unForkedCompressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + LZ4Compressor compressor = ESLZ4Compressor.INSTANCE; + int forkedCompressedSize = compressor.compress(uncompressed, compressed); + LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor(); + int unForkedCompressedSize = unForkedCompressor.compress(uncompressed, unForkedCompressed); + assertEquals(unForkedCompressedSize, forkedCompressedSize); + assertArrayEquals(compressed, unForkedCompressed); + + LZ4FastDecompressor decompressor = LZ4Factory.safeInstance().fastDecompressor(); + byte[] output = new byte[uncompressed.length]; + decompressor.decompress(compressed, output); + + assertArrayEquals(uncompressed, output); + } + } + + public void testCompressRandomIntBytes() throws IOException { + for (int i = 0; i < 15; ++i) { + int uncompressedBytesLength = randomFrom(16, 32, 64, 128, 256, 512, 1024) * 1024; + + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(uncompressedBytesLength); + for (int j = 0; j < uncompressedBytesLength / 4; ++j) { + bytesStreamOutput.writeInt(randomFrom(0, 1, randomInt())); + } + byte[] uncompressed = new byte[uncompressedBytesLength]; + bytesStreamOutput.bytes().streamInput().read(uncompressed); + + byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + byte[] unForkedCompressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + LZ4Compressor compressor = ESLZ4Compressor.INSTANCE; + int forkedCompressedSize = compressor.compress(uncompressed, compressed); + LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor(); + int unForkedCompressedSize = unForkedCompressor.compress(uncompressed, unForkedCompressed); + assertEquals(unForkedCompressedSize, forkedCompressedSize); + assertArrayEquals(unForkedCompressed, compressed); + + LZ4FastDecompressor decompressor = LZ4Factory.safeInstance().fastDecompressor(); + byte[] output = new byte[uncompressed.length]; + decompressor.decompress(compressed, output); + + assertArrayEquals(uncompressed, output); + } + } +} diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java new file mode 100644 index 0000000000000..79607de5140bb --- /dev/null +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +public class ESLZ4DecompressorTests extends ESTestCase { + + public void testDecompressRealisticUnicode() { + for (int i = 0; i < 15; ++i) { + int stringLengthMultiplier = randomFrom(5, 10, 20, 40, 80, 160, 320); + + final String uncompressedString = randomRealisticUnicodeOfCodepointLength(stringLengthMultiplier * 1024); + byte[] uncompressed = uncompressedString.getBytes(StandardCharsets.UTF_8); + + byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + LZ4Compressor compressor = LZ4Factory.safeInstance().fastCompressor(); + int unForkedDestinationBytes = compressor.compress(uncompressed, compressed); + + LZ4FastDecompressor decompressor = ESLZ4Decompressor.INSTANCE; + byte[] output = new byte[uncompressed.length]; + int forkedDestinationBytes = decompressor.decompress(compressed, output); + + assertEquals(unForkedDestinationBytes, forkedDestinationBytes); + assertArrayEquals(uncompressed, output); + } + } + + public void testDecompressRandomBytes() throws IOException { + for (int i = 0; i < 15; ++i) { + int uncompressedBytesLength = randomFrom(16, 32, 64, 128, 256, 512, 1024) * 1024; + + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(uncompressedBytesLength); + for (int j = 0; j < uncompressedBytesLength / 4; ++j) { + bytesStreamOutput.writeInt(randomFrom(0, 1, randomInt())); + } + byte[] uncompressed = new byte[uncompressedBytesLength]; + bytesStreamOutput.bytes().streamInput().read(uncompressed); + + byte[] compressed = new byte[uncompressed.length + uncompressed.length / 255 + 16]; + LZ4Compressor compressor = LZ4Factory.safeInstance().fastCompressor(); + int unForkedDestinationBytes = compressor.compress(uncompressed, compressed); + + LZ4FastDecompressor decompressor = ESLZ4Decompressor.INSTANCE; + byte[] output = new byte[uncompressed.length]; + int forkedDestinationBytes = decompressor.decompress(compressed, output); + + assertEquals(unForkedDestinationBytes, forkedDestinationBytes); + assertArrayEquals(uncompressed, output); + } + } +} diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4Tests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4Tests.java new file mode 100644 index 0000000000000..46f675f7ca56c --- /dev/null +++ b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4Tests.java @@ -0,0 +1,510 @@ +/* + * Copyright 2020 Adrien Grand and the lz4-java contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ + +package org.elasticsearch.lz4; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Exception; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.jpountz.lz4.LZ4SafeDecompressor; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file + * net.jpountz.lz4.LZ4Test. + * + * It modifies the test case to remove unneeded tests (safe decompressor, native libs, etc). Additionally, + * we only test our compressor/decompressor and the pure java "safe" lz4-java compressor/decompressor. + * Finally, on any "round-trip" tests we compress data using the safe lz4-java instance and compare that the + * compression is the same as the compressor instance we are testing. + */ +public class ESLZ4Tests extends AbstractLZ4TestCase { + + // Modified to only test ES decompressor instances + static LZ4FastDecompressor[] FAST_DECOMPRESSORS = new LZ4FastDecompressor[] { + ESLZ4Decompressor.INSTANCE + }; + + // Modified to not test any SAFE_DECOMPRESSORS, as we do not support it + static LZ4SafeDecompressor[] SAFE_DECOMPRESSORS = new LZ4SafeDecompressor[0]; + + // Modified to delete testMaxCompressedLength which requires native library. Additionally, we do not + // modify the maxCompressedLength logic + + private static byte[] getCompressedWorstCase(byte[] decompressed) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len = decompressed.length; + if (len >= LZ4Constants.RUN_MASK) { + baos.write(LZ4Constants.RUN_MASK << LZ4Constants.ML_BITS); + len -= LZ4Constants.RUN_MASK; + while (len >= 255) { + baos.write(255); + len -= 255; + } + baos.write(len); + } else { + baos.write(len << LZ4Constants.ML_BITS); + } + try { + baos.write(decompressed); + } catch (IOException e) { + throw new AssertionError(); + } + return baos.toByteArray(); + } + + public void testEmpty() { + testRoundTrip(new byte[0]); + } + + public void testUncompressWorstCase(LZ4FastDecompressor decompressor) { + final int len = randomInt(100 * 1024); + final int max = randomIntBetween(1, 255); + byte[] decompressed = randomArray(len, max); + byte[] compressed = getCompressedWorstCase(decompressed); + byte[] restored = new byte[decompressed.length]; + int cpLen = decompressor.decompress(compressed, 0, restored, 0, decompressed.length); + assertEquals(compressed.length, cpLen); + assertArrayEquals(decompressed, restored); + } + + public void testUncompressWorstCase() { + for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) { + testUncompressWorstCase(decompressor); + } + } + + public void testUncompressWorstCase(LZ4SafeDecompressor decompressor) { + final int len = randomInt(100 * 1024); + final int max = randomIntBetween(1, 256); + byte[] decompressed = randomArray(len, max); + byte[] compressed = getCompressedWorstCase(decompressed); + byte[] restored = new byte[decompressed.length]; + int uncpLen = decompressor.decompress(compressed, 0, compressed.length, restored, 0); + assertEquals(decompressed.length, uncpLen); + assertArrayEquals(decompressed, restored); + } + + // Modified to delete testUncompressSafeWorstCase (we do not have "safe" decompressor) + + // Modified to only test 1 (fast) decompressor + public void testRoundTrip(byte[] data, int off, int len, + LZ4Compressor compressor, + LZ4FastDecompressor decompressor) { + for (Tester tester : Arrays.asList(Tester.BYTE_ARRAY, Tester.BYTE_BUFFER, Tester.BYTE_ARRAY_WITH_LENGTH, + Tester.BYTE_BUFFER_WITH_LENGTH)) { + testRoundTrip(tester, data, off, len, compressor, decompressor); + } + if (data.length == len && off == 0) { + for (SrcDestTester tester : Arrays.asList(SrcDestTester.BYTE_ARRAY, SrcDestTester.BYTE_BUFFER, + SrcDestTester.BYTE_ARRAY_WITH_LENGTH, + SrcDestTester.BYTE_BUFFER_WITH_LENGTH)) { + testRoundTrip(tester, data, compressor, decompressor); + } + } + } + + // Modified to only test 1 (fast) decompressor + public void testRoundTrip( + Tester tester, + byte[] data, int off, int len, + LZ4Compressor compressor, + LZ4FastDecompressor decompressor) { + final int maxCompressedLength = tester.maxCompressedLength(len); + // "maxCompressedLength + 1" for the over-estimated compressed length test below + final T compressed = tester.allocate(maxCompressedLength + 1); + final int compressedLen = tester.compress(compressor, + tester.copyOf(data), off, len, + compressed, 0, maxCompressedLength); + + // Modified to compress using an unforked lz4-java compressor and verify that the results are same. + T expectedCompressed = tester.allocate(maxCompressedLength + 1); + LZ4Compressor unForkedCompressor = LZ4Factory.safeInstance().fastCompressor(); + final int expectedCompressedLen = tester.compress(unForkedCompressor, + tester.copyOf(data), off, len, + expectedCompressed, 0, maxCompressedLength); + assertEquals(expectedCompressedLen, compressedLen); + assertArrayEquals(tester.copyOf(expectedCompressed, 0, expectedCompressedLen), tester.copyOf(compressed, 0, compressedLen)); + + + // test decompression + final T restored = tester.allocate(len); + assertEquals(compressedLen, tester.decompress(decompressor, compressed, 0, restored, 0, len)); + assertArrayEquals(Arrays.copyOfRange(data, off, off + len), tester.copyOf(restored, 0, len)); + + // make sure it fails if the compression dest is not large enough + tester.fill(restored, randomByte()); + final T compressed2 = tester.allocate(compressedLen-1); + try { + final int compressedLen2 = tester.compress(compressor, + tester.copyOf(data), off, len, + compressed2, 0, compressedLen - 1); + // Compression can succeed even with the smaller dest + // because the compressor is allowed to return different compression results + // even when it is invoked with the same input data. + // In this case, just make sure the compressed data can be successfully decompressed. + assertEquals(compressedLen2, tester.decompress(decompressor, compressed2, 0, restored, 0, len)); + assertArrayEquals(Arrays.copyOfRange(data, off, off + len), tester.copyOf(restored, 0, len)); + } catch (LZ4Exception e) { + // OK + } + + if (tester != Tester.BYTE_ARRAY_WITH_LENGTH && tester != Tester.BYTE_BUFFER_WITH_LENGTH) { + // LZ4DecompressorWithLength will succeed in decompression + // because it ignores destLen. + + if (len > 0) { + // decompression dest is too small + try { + tester.decompress(decompressor, compressed, 0, restored, 0, len - 1); + fail(); + } catch (LZ4Exception e) { + // OK + } + } + + // decompression dest is too large + final T restored2 = tester.allocate(len+1); + try { + final int cpLen = tester.decompress(decompressor, compressed, 0, restored2, 0, len + 1); + fail("compressedLen=" + cpLen); + } catch (LZ4Exception e) { + // OK + } + } + + // Modified to delete "safe" decompressor tests + } + + // Modified to only test 1 (fast) decompressor + public void testRoundTrip(SrcDestTester tester, + byte[] data, + LZ4Compressor compressor, + LZ4FastDecompressor decompressor) { + final T original = tester.copyOf(data); + final int maxCompressedLength = tester.maxCompressedLength(data.length); + final T compressed = tester.allocate(maxCompressedLength); + final int compressedLen = tester.compress(compressor, + original, + compressed); + if (original instanceof ByteBuffer) { + assertEquals(data.length, ((ByteBuffer)original).position()); + assertEquals(compressedLen, ((ByteBuffer)compressed).position()); + ((ByteBuffer)original).rewind(); + ((ByteBuffer)compressed).rewind(); + } + + // test decompression + final T restored = tester.allocate(data.length); + assertEquals(compressedLen, tester.decompress(decompressor, compressed, restored)); + if (original instanceof ByteBuffer) { + assertEquals(compressedLen, ((ByteBuffer)compressed).position()); + assertEquals(data.length, ((ByteBuffer)restored).position()); + } + assertArrayEquals(data, tester.copyOf(restored, 0, data.length)); + if (original instanceof ByteBuffer) { + ((ByteBuffer)compressed).rewind(); + ((ByteBuffer)restored).rewind(); + } + + // Modified to delete "safe" decompressor tests + } + + // Modified to delete unnecessary method + + public void testRoundTrip(byte[] data, int off, int len) { + // Modified to only test safe instance and forked instance + for (LZ4Compressor compressor : Arrays.asList(LZ4Factory.safeInstance().fastCompressor(), ESLZ4Compressor.INSTANCE)) { + for (LZ4FastDecompressor decompressor : Arrays.asList(LZ4Factory.safeInstance().fastDecompressor(), + ESLZ4Decompressor.INSTANCE)) { + testRoundTrip(data, off, len, compressor, decompressor); + } + } + } + + public void testRoundTrip(byte[] data) { + testRoundTrip(data, 0, data.length); + } + + public void testRoundTrip(String resource) throws IOException { + final byte[] data = readResource(resource); + testRoundTrip(data); + } + + public void testRoundtripGeo() throws IOException { + // Modified path to point at resource + testRoundTrip("calgary/geo.binary"); + } + + public void testRoundtripBook1() throws IOException { + // Modified path to point at resource + testRoundTrip("calgary/book1"); + } + + public void testRoundtripPic() throws IOException { + // Modified path to point at resource + testRoundTrip("calgary/pic.binary"); + } + + public void testNullMatchDec() { + // 1 literal, 4 matchs with matchDec=0, 8 literals + final byte[] invalid = new byte[] { 16, 42, 0, 0, (byte) 128, 42, 42, 42, 42, 42, 42, 42, 42 }; + // decompression should neither throw an exception nor loop indefinitely + for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) { + decompressor.decompress(invalid, 0, new byte[13], 0, 13); + } + for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) { + decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0); + } + } + + public void testEndsWithMatch() { + // 6 literals, 4 matchs + final byte[] invalid = new byte[] { 96, 42, 43, 44, 45, 46, 47, 5, 0 }; + final int decompressedLength = 10; + + for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) { + try { + // it is invalid to end with a match, should be at least 5 literals + decompressor.decompress(invalid, 0, new byte[decompressedLength], 0, decompressedLength); + assertTrue(decompressor.toString(), false); + } catch (LZ4Exception e) { + // OK + } + } + + for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) { + try { + // it is invalid to end with a match, should be at least 5 literals + decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0); + assertTrue(false); + } catch (LZ4Exception e) { + // OK + } + } + } + + public void testEndsWithLessThan5Literals() { + // 6 literals, 4 matchs + final byte[] invalidBase = new byte[] { 96, 42, 43, 44, 45, 46, 47, 5, 0 }; + + for (int i = 1; i < 5; ++i) { + final byte[] invalid = Arrays.copyOf(invalidBase, invalidBase.length + 1 + i); + invalid[invalidBase.length] = (byte) (i << 4); // i literals at the end + + for (LZ4FastDecompressor decompressor : FAST_DECOMPRESSORS) { + try { + // it is invalid to end with a match, should be at least 5 literals + decompressor.decompress(invalid, 0, new byte[20], 0, 20); + assertTrue(decompressor.toString(), false); + } catch (LZ4Exception e) { + // OK + } + } + + for (LZ4SafeDecompressor decompressor : SAFE_DECOMPRESSORS) { + try { + // it is invalid to end with a match, should be at least 5 literals + decompressor.decompress(invalid, 0, invalid.length, new byte[20], 0); + assertTrue(false); + } catch (LZ4Exception e) { + // OK + } + } + } + } + + // Modified to delete testWriteToReadOnlyBuffer. We only compress to byte arrays so this test is + // unnecessary + + public void testAllEqual() { + // Modified to not use @Repeat + for (int i = 0; i < 5; ++i) { + final int len = randomBoolean() ? randomInt(20) : randomInt(100000); + final byte[] buf = new byte[len]; + Arrays.fill(buf, randomByte()); + testRoundTrip(buf); + } + } + + public void testMaxDistance() { + final int len = randomIntBetween(1 << 17, 1 << 18); + final int off = randomInt(len - (1 << 16) - (1 << 15)); + final byte[] buf = new byte[len]; + for (int i = 0; i < (1 << 15); ++i) { + buf[off + i] = randomByte(); + } + System.arraycopy(buf, off, buf, off + 65535, 1 << 15); + testRoundTrip(buf); + } + + public void testRandomData() { + // Modified to not use @Repeat + for (int i = 0; i < 10; ++i) { + final int n = randomIntBetween(1, 15); + final int off = randomInt(1000); + final int len = randomBoolean() ? randomInt(1 << 16) : randomInt(1 << 20); + final byte[] data = randomArray(off + len + randomInt(100), n); + testRoundTrip(data, off, len); + } + } + + // https://github.com/jpountz/lz4-java/issues/12 + public void testRoundtripIssue12() { + byte[] data = new byte[]{ + 14, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 72, 14, 72, 14, 85, 3, 72, 14, 72, 14, 72, 14, 72, 14, 72, 14, 72, 14, 85, 3, 72, + 14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 85, 3, 72, 14, 50, 64, 0, 46, -1, 0, 0, 0, 29, 3, 85, + 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, + 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, + 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, + 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 50, 64, 0, 47, -105, 0, 0, 0, 30, 3, -97, 6, 0, 68, -113, + 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, + 8, -113, 0, 68, -97, 3, 0, 2, -97, 6, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, + 6, 0, 68, -113, 0, 120, 64, 0, 48, 4, 0, 0, 0, 31, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, + 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, + 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, + 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, + 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, + 41, 72, 32, 72, 18, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 39, 24, 32, 34, 124, 0, 120, 64, 0, 48, 80, 0, 0, 0, 31, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, + 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, + 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, + 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, + 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, + 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, + 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, + 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, + 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, + 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, + 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 72, 34, 72, + 29, 72, 37, 72, 35, 72, 45, 72, 23, 72, 46, 72, 20, 72, 40, 72, 33, 72, 25, 72, 39, 72, 38, 72, 26, 72, 28, 72, 42, 72, 24, 72, + 27, 72, 36, 72, 41, 72, 32, 72, 18, 72, 30, 72, 22, 72, 31, 72, 43, 72, 19, 50, 64, 0, 49, 20, 0, 0, 0, 32, 3, -97, 6, 0, + 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, + 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, + 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, + 3, -97, 6, 0, 50, 64, 0, 50, 53, 0, 0, 0, 34, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97, + 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, + -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, + 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, + 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, + 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, + -97, 6, 0, 50, 64, 0, 51, 85, 0, 0, 0, 36, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, + 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, -97, 5, 0, 2, 3, 85, 8, -113, 0, 68, + -97, 3, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, + 68, -113, 0, 2, 3, -97, 6, 0, 50, -64, 0, 51, -45, 0, 0, 0, 37, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, + 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, -97, 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -113, 0, 2, 3, -97, + 6, 0, 68, -113, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 2, 3, 85, 8, -113, 0, 68, -97, 3, 0, 120, 64, 0, 52, -88, 0, 0, + 0, 39, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, + 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, + 5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, + 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, + 5, 72, 13, 85, 5, 72, 13, 72, 13, 72, 13, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, 72, + 13, 85, 5, 72, 13, 72, 13, 85, 5, 72, 13, -19, -24, -101, -35 + }; + testRoundTrip(data, 9, data.length - 9); + } + + private static void assertCompressedArrayEquals(String message, byte[] expected, byte[] actual) { + int off = 0; + int decompressedOff = 0; + while (true) { + if (off == expected.length) { + break; + } + final Sequence sequence1 = readSequence(expected, off); + final Sequence sequence2 = readSequence(actual, off); + assertEquals(message + ", off=" + off + ", decompressedOff=" + decompressedOff, sequence1, sequence2); + off += sequence1.length; + decompressedOff += sequence1.literalLen + sequence1.matchLen; + } + } + + private static Sequence readSequence(byte[] buf, int off) { + final int start = off; + final int token = buf[off++] & 0xFF; + int literalLen = token >>> 4; + if (literalLen >= 0x0F) { + int len; + while ((len = buf[off++] & 0xFF) == 0xFF) { + literalLen += 0xFF; + } + literalLen += len; + } + off += literalLen; + if (off == buf.length) { + return new Sequence(literalLen, -1, -1, off - start); + } + int matchDec = (buf[off++] & 0xFF) | ((buf[off++] & 0xFF) << 8); + int matchLen = token & 0x0F; + if (matchLen >= 0x0F) { + int len; + while ((len = buf[off++] & 0xFF) == 0xFF) { + matchLen += 0xFF; + } + matchLen += len; + } + matchLen += 4; + return new Sequence(literalLen, matchDec, matchLen, off - start); + } + + private static class Sequence { + final int literalLen, matchDec, matchLen, length; + + private Sequence(int literalLen, int matchDec, int matchLen, int length) { + this.literalLen = literalLen; + this.matchDec = matchDec; + this.matchLen = matchLen; + this.length = length; + } + + @Override + public String toString() { + return "Sequence [literalLen=" + literalLen + ", matchDec=" + matchDec + + ", matchLen=" + matchLen + "]"; + } + + @Override + public int hashCode() { + return 42; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Sequence other = (Sequence) obj; + if (literalLen != other.literalLen) + return false; + if (matchDec != other.matchDec) + return false; + if (matchLen != other.matchLen) + return false; + return true; + } + + } +} diff --git a/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/README b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/README new file mode 100644 index 0000000000000..a5fd267b2fe2c --- /dev/null +++ b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/README @@ -0,0 +1 @@ +Data from the Calgary corpus (http://corpus.canterbury.ac.nz/descriptions/#calgary). diff --git a/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/book1 b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/book1 new file mode 100644 index 0000000000000..5cbadedbf4ea5 --- /dev/null +++ b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/book1 @@ -0,0 +1,1377 @@ + + + + +

+DESCRIPTION OF FARMER OAK -- AN INCIDENT +When Farmer Oak smiled, the corners of his mouth +spread till they were within an unimportant distance of +his ears, his eyes were reduced to chinks, and diverging +wrinkles appeared round them, extending upon his +countenance like the rays in a rudimentary sketch of +the rising sun. +His Christian name was Gabriel, and on working +days he was a young man of sound judgment, easy +motions, proper dress, and general good character. On +Sundays he was a man of misty views, rather given to +postponing, and hampered by his best clothes and +umbrella : upon the whole, one who felt himself to +occupy morally that vast middle space of Laodicean +neutrality which lay between the Communion people +of the parish and the drunken section, -- that is, he went +to church, but yawned privately by the time the con+ +gegation reached the Nicene creed,- and thought of +what there would be for dinner when he meant to be +listening to the sermon. Or, to state his character as +it stood in the scale of public opinion, when his friends +and critics were in tantrums, he was considered rather a +bad man ; when they were pleased, he was rather a good +man ; when they were neither, he was a man whose +moral colour was a kind of pepper-and-salt mixture. +Since he lived six times as many working-days as +Sundays, Oak's appearance in his old clothes was most +peculiarly his own -- the mental picture formed by his +neighbours in imagining him being always dressed in +that way. He wore a low-crowned felt hat, spread out +at the base by tight jamming upon the head for security +in high winds, and a coat like Dr. Johnson's ; his lower +extremities being encased in ordinary leather leggings +and boots emphatically large, affording to each foot a +roomy apartment so constructed that any wearer might +stand in a river all day long and know nothing of +

+damp -- their maker being a conscientious man who +endeavoured to compensate for any weakness in his cut +by unstinted dimension and solidity. +Mr. Oak 'carried 'about him, by way of watch,+ +what may be called a small silver clock; in other +words, it was a watch as to shape and intention, and +a small clock as to size. This instrument being several +years older than Oak's grandfather, had the peculiarity +of going either too fast or not at all. The smaller +of its hands, too, occasionally slipped round on the +pivot, and thus, though the minutes were told with +precision, nobody could be quite certain of the hour +they belonged to. The stopping peculiarity of his +watch Oak remedied by thumps and shakes, and he +escaped any evil consequences from the other two +defects by constant comparisons with and observations +of the sun and stars, and by pressing his face close +to the glass of his neighbours' windows, till he could +discern the hour marked by the green-faced timekeepers +within. It may be mentioned that Oak's fob being +difficult of access, by reason of its somewhat high +situation in the waistband of his trousers (which also +lay at a remote height under his waistcoat), the watch +was as a necessity pulled out by throwing the body to +one-side, compressing the- mouth and face to a mere +mass of- ruddy flesh- on account -of the exertion, and +drawing up the watch by its chain, like a bucket from a +well. +But some thoughtfull persons, who had seen him +walking across one of his fields on a certain December +morning -- sunny and exceedingly mild -- might have +regarded Gabriel Oak in other aspects than these. In +his face one might notice that many of the hues and +curves of youth had tarried on to manhood: there even +remained in his remoter crannies some relics of the boy. +His height and breadth would have been sufficient to +make his presence imposing, had they been exhibited +with due consideration. But there is a way some men +have, rural and urban alike, for which the mind is more +responsible than flesh and sinew : it is a way of curtail+ +ing their dimensions by their manner of showing them. +And from a quiet modesty that would have become a +vestal which seemed continually to impress upon him +

+that he had no great claim on the world's room, Oak +walked unassumingly and with a faintly perceptible +bend, yet distinct from a bowing of the shoulders. +This may be said to be a defect in an individual if he +depends for his valuation more upon his appearance +than upon his capacity to wear well, which Oak did not. +He had just reached the time of life at which " young' +is ceasing to be the prefix of "man ' in speaking of one. +He was at the brightest period of masculine growth, +for his intellect and his emotions were clearly separated : +he had passed the time during which the influence of +youth indiscriminately mingles them in the character +of impulse, and he had not yet arrived at the stage +wherein they become united again, in the character of +prejudice, by the influence of a wife and family. In +short, he was twenty-eight, and a bachelor. +The field he was in this morning sloped to a +ridge called Norcombe Hill. Through a spur of this +hill ran the highway between Emminster and Chalk+ +Newton. Casually glancing over the hedge, Oak saw +coming down the incline before him an ornamental +spring waggon, painted yellow and gaily marked, +drawn by two horses, a waggoner walking alongside +bearing a whip perpendicularly. The waggon was +laden with household goods and window plants, and +on the apex of the whole sat a woman, 'young-'and +attractive. Gabriel had not beheld the sight for more +than half a minute, when the vehicle was brought to a +standstill just beneath his eyes. +" The tailboard of the waggon is gone, Miss,' said the +waggoner. +"Then I heard it fall,' said the girl, in a soft, though +not particularly low voice. "I heard a noise I could +not account for when we were coming up the hill.' +"I'll run back.' + +" Do,' she answered. + +The sensible horses stood -- perfectly still, and the +waggoner's steps sank fainter and fainter in the distance. +The girl on the summit of the load sat motionless, +surrounded by tables and chairs with their legs upwards, +backed by an oak settle, and ornamented in front by +pots of geraniums, myrtles, and cactuses, together with +

+a caged canary -- all probably from the windows of the +house just vacated. There was also a cat in a willow +basket, from the partly-opened lid of which she gazed +with half-closed eyes, and affectionately-surveyed the +small birds around. +The handsome girl waited for some time idly in her +place, and the only sound heard in the stillness-was -the +hopping of the canary up-and down the perches of its +prison. Then she looked attentively downwards. It +was not at the bird, nor at the cat; it was at an oblong +package tied in paper, and lying between them. She +turned her head to learn if the waggoner were coming. +He was not yet in sight; and her-eyes crept back to +the package, her thoughts seeming to run 'upon what +was inside it. At length she drew the article into her +lap, and untied the paper covering; a small swing +looking-glass was disclosed, in which she proceeded to +survey herself attentively. She parted her lips and +smiled. +It was a fine morning, and the sun lighted up to a +scarlet glow the crimson jacket she wore, and painted +a soft lustre upon her bright face and dark hair. The +myrtles, geraniums, and cactuses packed around her +were fresh and green, and at such a leafless season they +invested the whole concern of horses, waggon, furniture, +and girl with a peculiar vernal charm. What possessed +her to indulge in such a performance in the sight of the +sparrows, blackbirds, and unperceived farmer who were +alone its spectators, -- whether the smile began as a +factitious one, to test her capacity in that art, -- nobody +knows ; it ended certainly in a real smile. She blushed +at herself, and seeing her reflection blush, blushed the +more. +The change from the customary spot and necessary +occasion of such an act -- from the dressing hour in a +bedroom to a time of travelling out of doors -- lent to +the idle deed a novelty it did not intrinsically possess. +The picture was a delicate one. Woman's prescriptive +infirmity had stalked into the sunlight, which had +clothed it in the freshness of an originality. A +cynical inference was irresistitle by Gabriel Oak as he +regarded the scene, generous though he fain would have +been. There was no necessity whatever for her looking +in the glass. She did not adjust her hat, or pat her +

+hair, or press a dimple into shape, or do one thing to +signify that any such intention had been her motive in +taking up the glass. She simply observed herself as a +fair product of Nature in the feminine kind, her thoughts +seeming to glide into far-off though likely dramas in +which men would play a part -- vistas of probable +triumphs -- the smiles being of a phase suggesting that +hearts were imagined as lost and won. Still, this was +but conjecture, and the whole series of actions was so +idly put forth as to make it rash to assert that intention +had any part in them at all. +The waggoner's steps were heard returning. She +put the glass in the paper, and the whole again into its +place. +When the waggon had passed on, Gabriel withdrew +from his point of espial, and descending into the road, +followed the vehicle to the turnpike-gate some way +beyond the bottom of the hill, where the object of his +contemplation now halted for the payment of toll. About +twenty steps still remained between him and the gate, +when he heard a dispute. lt was a difference con+ +cerning twopence between the persons with the waggon +and the man at the toll-bar. +" Mis'ess's niece is upon the top of the things, and +she says that's enough that I've offered ye, you great +miser, and she won't pay any more.' These were the +waggoner's words. +"Very well ; then mis'ess's niece can't pass,' said the +turnpike-keeper, closing the gate. +Oak looked from one to the other of the disputants, +and fell into a reverie. There was something in the +tone of twopence remarkably insignificant. Threepence +had a definite value as money -- it was an appreciable +infringement on a day's wages, and, as such, a higgling +matter ; but twopence -- -- " Here,' he said, stepping +forward and handing twopence to the gatekeeper ; "let +the young woman pass.' He looked up at her then; +she heard his words, and looked down. +Gabriel's features adhered throughout their form so +exactly to the middle line between the beauty of St. +John and the ugliness of Judas Iscariot, as represented +in a window of the church he attended, that not a single +lineament could be selected and called worthy either of +distinction or notoriety. The redjacketed and dark+ +

+haired maiden seemed to think so too, for she carelessly +glanced over him, and told her man to drive on. She +might have looked her thanks to Gabriel on a minute +scale, but she did not speak them; more probably she +felt none, for in gaining her a passage he had lost her +her point, and we know how women take a favour of +that kind. +The gatekeeper surveyed the retreating vehicle. +" That's a handsome maid ' he said to Oak +" But she has her faults,' said Gabriel. +" True, farmer. ' +"And the greatest of them is -- well, what it is +always.' +" Beating people down ? ay, 'tis so.' +"O no.' +" What, then ? ' +Gabriel, perhaps a little piqued by the comely +traveller's indifference, glanced back to where he had +witnessed her performance over the hedge, and said, +" Vanity.' + +

+NIGHT -- THE FLOCK -- AN INIERIOR -- ANOTHER INTERIOR +IT was nearly midnight on the eve of St. Thomas"s, the +shortest day in the year. A desolating wind wandered +from the north over the hill whereon Oak had watched +the yellow waggon and its occupant in the sunshine of +a few days earlier. + Norcombe Hill -- not far from lonely Toller-Down + -- was one of the spots which suggest to a passer-by +that he is in the presence of a shape approaching the +indestructible as nearly as any to be found on earth. +It was a featureless convexity of chalk and soil -- an +ordinary specimen of those smoothly-outlined protuber+ +ances of the globe which may remain undisturbed on +some great day of confusion, when far grander heights +and dizzy granite precipices topple down. +The hill was covered on its northern side by an +ancient and decaying plantation of beeches, whose +upper verge formed a line over the crest, fringing its +arched curve against the sky, like a mane. To-night +these trees sheltered the southern slope from the keenest +blasts, which smote the wood and floundered through +it with a sound as of grumbling, or gushed over its +crowning boughs in a weakened moan. The dry leaves +in the ditch simmered and boiled in the same breezes, +a tongue of air occasionally ferreting out a few, and +sending them spinning across the grass. A group or +two of the latest in date amongst the dead multitude +had remained till this very mid-winter time on the twigs +which bore them and in falling rattled against the trunks +with smart taps: +Betwenne this half-wooded, half naked hill, and the +vague still horizon that its summit indistinctly com+ +manded, was a mysterious sheet of fathomless shade + -- the sounds from which suggested that what it con+ +cealed bore some reduced resemblance to features here. +

+The thin grasses, more or less coating the hill, were +touched by the wind in breezes of differing powers, and +almost of differing natures -- one rubbing the blades +heavily, another raking them piercingly, another brushing +them like a soft broom. The instinctive act of human+ +kind was to stand and listen, and learn how the trees +to each other in the regular antiphonies of a cathedral +choir; how hedges and other shapes to leeward them +caught the note, lowering it to the tenderest sob; and +how the hurrying gust then plunged into the south, to +be heard no more. +The sky was clear -- remarkably clear -- and the +twinkling of all the stars seemed to be but throbs of +one body, timed by a common pulse. The North Star +was directly in the wind's eye, and since evening the +Bear had swung round it outwardly to the east, till he +was now at a right angle with the meridian. A +difference of colour in the stars -- oftener read of than +seen in England-was really perceptible here. The +sovereign brilliancy of Sirius pierced the eye with a steely +glitter, the star called Capella was yellow, Aldebaran and +Betelgueux shone with a fiery red. +To persons standing alone on a hill during a clear +midnight such as this, the roll of the world eastward is +almost a palpable movement. The sensation may be +caused by the panoramic glide of the stars past earthly +objects, which is perceptible in a few minutes of still+ +ness, or by the better outlook upon space that a hill +affords, or by the wind, or by the solitude ; but whatever +be its origin, the impression of riding along is vivid and +abiding. The poetry of motion is a phrase much in +use, and to enjoy the epic form of that gratification it +is necessary to stand on a hill at a small hour of the +night, and, having first expanded with a sense of differ+ +ence from the mass of civilised mankind, who are +dreamwrapt and disregardful of all such proceedings at +this time, long and quietly watch your stately progress +through the stars. After such a nocturnal reconnoitre +it is hard to get back to earth, and to believe that the +consciousness of such majestic speeding is derived from +a tiny human frame. +Suddenly an unexpected series of sounds began to +

+be heard +in this place up against the sky. They had a +clearness which was to be found nowhere in the wind, +and a sequence which was to be found nowhere in +nature. They were the notes of Farmer Oak's flute. +The tune was not floating unhindered into the open +air : it seemed muffled in some way, and was altogether +too curtailed in power to spread high or wide. It came +from the direction of a small dark object under the +plantation hedge -- a shepherd's hut -- now presenting +an outline to which an uninitiated person might have +been puzzled to attach either meaning or use. +The image as a whole was that of a small Noah's +Ark on a small Ararat, allowing the traditionary outlines +and general form of the Ark which are followed by toy+ +makers -- and by these means are established in men's +imaginations among their firmest, because earliest im+ +pressions -- to pass as an approximate pattern. The +hut stood on little wheels, which raised its floor about a +foot from the ground. Such shepherds' huts are dragged +into the fields when the lambing season comes on, to +shelter the shepherd in his- enforced nightly attendance. +It was only latterly that people had begun to call +Gabriel !Farmer' Oak. During the twelvemonth pre+ +ceding this time he had been enabled by sustained +efforts of industry and chronic good spirits to lease the +small shepp farm of which Norcombe Hill was a portion, +and stock it with two hundred sheep. Previously he +had been a bailiff for a short time, and earlier still a +shepherd only, having from his childhood assisted his +father in tending the floeks of large proprietors, till old +Gabriel sank to rest. +This venture, unaided and alone, into the paths of +farming as master and not as man, with an advance of +sheep not yet paid for, was a critical juncture with +Gabriel Oak, and he recognised his position clearly. +The first movement in his new progress was the lambing +of his ewes, and sheep having been his speciality from +his "youth, he wisely refrained from deputing -- the task +of tending them at this season to a hireling or a novice. +The wind continued to beat-about the corners of the +hut, but the flute-playing ceased. A rectangular space +of light +

+appeared in the side of the hut, and in the +opening the outline of Farmer Oak's figure. He carried +a lantern in his hand, and closing the door behind him, +came forward and busied himself about this nook of the +field for nearly twenty minutes, the lantern light appear+ +ing and disappearing here and there, and brightening +him or darkening him as he stood before or behind it. +Oak's motions, though they had a quiet-energy, were +slow, and their deliberateness accorded well with his +occupation. Fitness being the basis of beauty, nobody +could-have denied that his steady swings and turns" +in and- about the flock had elements of grace, Yet, +although if occasion demanded he could do or think a +thing with as mercurial a dash as can the men of towns +who are more to the manner born, his special power, +morally, physically, and mentally, was static, owing +little or nothing to momentum as a rule. +A close examination of the ground hereabout, even +by the wan starlight only, revealed how a portion of +what would have been casually called a wild slope had +been appropriated by Farmer Oak for his great purpose +this winter. Detached hurdles thatched with straw +were stuck into the ground at various scattered points, +amid and under which the whitish forms of his meek +ewes moved and rustled. The ring of the sheep-bell, +which had been silent during his absence, recommenced, +in tones that had more mellowness than clearness, owing +to an increasing growth of surrounding wool. This +continued till Oak withdrew again from the flock. He + -- returned to the hut, bringing in his arms a new-born +lamb, consisting of four legs large enough for a full+ +grown sheep, united by a seemingly inconsiderable mem+ +brane about half the substance of the legs collectively, +which constituted the animal's entire body just at present. +The little speck of life he placed on a wisp of hay +before the small stove, where a can of milk was simmer+ +ing. Oak extinguished the lantern by blowing into it +and then pinching the snuff, the cot being lighted +by a candle suspended by a twisted wire. A rather +hard couch, formed of a few corn sacks thrown carelessly +down, covered half the floor of this little +

+habitation, and +here the young man stretched himself along, loosened +his woollen cravat, and closed his eyes. In about the +time a person unaccustomed to bodily labour would have +decided upon which side to lie, Farmer Oak was asleep. +The inside of the hut, as it now presented itself, was +cosy and alluring, and the scarlet handful of fire in +addition to the candle, reflecting its own genial colour +upon whatever it could reach, flung associations of +enjoyment even over utensils and tools. In the corner +stood the sheep-crook, and along a shelf at one side +were ranged bottles and canisters of the simple prepara+ +tions pertaining to ovine surgery and physic; spirits of +wine, turpentine, tar, magnesia, ginger, and castor-oil +being the chief. On a triangular shelf across the corner +stood bread, bacon, cheese, and a cup for ale or cider, +which was supplied from a flagon beneath. Beside the +provisions lay the flute whose notes had lately been +called forth by the lonely watcher to beguile a tedious +hour. The house was ventilated by two round holes, +like the lights of a ship's cabin, with wood slides+ +The lamb, revived by the warmth' began to bleat' +instant meaning, as expected sounds will. Passing +from the profoundest sleep to the most alert wakefulness +with the same ease that had accompanied the reverse +operation, he looked at his watch, found that the hour+ +hand had shifted again, put on his hat, took the lamb +in his arms, and carried it into the darkness. After +placing the little creature with its mother, he stood and +carefully examined the sky, to ascertain the time of +night from the altitudes of the stars. +The Dog-star and Aldebaran, pointing to the restless +Pleiades, were half-way up the Southern sky, and between +them hung Orion, which gorgeous constellation never +burnt more vividly than now, as it soared forth above +the rim of the landscape. Castor and Pollux will +the north-west; far away through the plantation Vega +and Cassiopeia's chair stood daintily poised on the +uppermost boughs. +

+"One o'clock,' said Gabriel. +Being a man not without a frequent consciousness +that there was some charm in this life he led, he stood +still after looking at the sky as a useful instrument, and +regarded it in an appreciative spirit, as a work of art +superlatively beautiful. For a moment he seemed +impressed with the speaking loneliness of the scene, or +rather with the complete abstraction from all its compass +of the sights and sounds of man. Human shapes,interferences, +troubles, and joys were all as if they were not, and there +seemed to be on the shaded hemisphere of the globe no sentient being +save himself; he could fancy them all gone round to the sunny side. + Occupied this, with eyes stretched afar, Oak gradually per+ +ceived that what he had previously taken to be a star low +down behind the outskirts of the plantation was in reality no +such thing. It was an artificial light, almost close at hand. + To find themselves utterly alone at night where company +is desirable and expected makes some people fearful; but a +case more trying by far to the nerves is to discover some +mysterious companionship when intuition, sensation, memory, +analogy, testimony, probability, induction -- every kind of +evidence in the logician's list -- have united to persuade con+ +sciousness that it is quite in isolation. + Farmer Oak went towards the plantation and pushed +through its lower boughs to the windy side. A dim mass under +the slope reminded him that a shed occupied a place here, +the site being a cutting into the slope of the hill, so that at +its back part the roof was almost level with the ground. In +front it was formed of board nailed to posts and covered with +tar as apreservative. Through crevices in the roof and side +spread streaks and spots of light, a combination of which made +the radiance that had attracted him. Oak stepped up behind, +where,leaning down upon the roof and putting his eye close +to a hole, he could see into the interior clearly. + The place contained two women and two cows. By the side +of the latter a steaming bran-mash stood in a bucket. One +of the women was past middle age. Her companion was ap+ +parently young and graceful; he could form no decided opinion +

+upon her looks, her position being almost beneath his eye, so +that he saw her in a bird's-eye view, as Milton's Satan first saw +Paradise. She wore no bonnet or het, but had enveloped her+ +self in a large cloak, which was carelessly flung over her head +as a covering. + "There, now we'll go home," said the elder of the two, resting + her knuckles upon her hips, and looking at their goings-on as +a whole. "I do hope Daisy will fetch round again now. I have +never been more frightened in my life, but I don't mind break+ +ing my rest if she recovers." + The young woman, whose eyelids were apparently inclined +to fall together on the smallest provocation of silence,yawned +in sympathy. + "I wish we were rich enough to pay a man to do these +things," she said. + "As we are not, we must do them ourselves," said the other; +"for you must help me if you stay." +"Well, my hat is gone, however," continued the younger. "It +went over the hedge, I think. The idea of such a slight wind +catching it." + The cow standing erect was of the Devon breed, and was +encased in a tight warm hide of rich Indian red, as absolutely +uniform from eyes to tail as if the animal had been dipped in +a dye of that colour, her long back being mathematically level. +The other was spotted,grey and white. Beside her Oak now +noticed a little calf about a day old, looking idiotically at +the two women, which showed that it had not long been +accustomed to the phenomenon of eyesight, and often turn+ +ing to the lantern, which it apparently mistook for the moon. +inherited instinct having as yet had little time for correction +by experience. Between the sheep and the cows Lucina had +been busy on Norcombe hill lately. + "I think we had better send for some oatmeal," said the +"Yes, aunt; and I'll ride over for it as soon as it is +light. ' +" But there's no side-saddle.' +

+"I can ride on the other : trust me.' +Oak, upon hearing these remarks, became more +curious to observe her features, but this prospect being +denied him by the hooding efect of the cloak, and by his +aerial position, he felt himself drawing upon his fancy +for their details. In making even horizontal and clear +inspections we colour and mould according to the warts +within us whatever our eyes bring in. Had Gabriel +been able from the first to get a distinct view of her + +countenance, his estimate of it as very handsome or +slightly so would have been as his soul required a +divinity at the moment or was ready supplied with one. +Having for some time known the want of a satisfactory +form to fill an increasing void within him, his position +moreover affording the widest scope for his fancy, he +painted her a beauty. +By one of those whimsical coincidences in which +Nature, like a busy mother, seems to spare a moment +from her unremitting labours to turn and make her +children smile, the girl now dropped the cloak, and +forth tumbled ropes of black hair over a red jacket. +Oak knew her instantly as the heroine of the yellow +waggon, myrtles, and looking-glass : prosily, as the +woman who owed him twopence. +They placed the calf beside its mother again, took +up the lantern, and went out, the light sinking down +the hill till it was no more than a nebula. Gabriel +Oak returned to his flock. + +

+A GIRL ON HORSEBACK -- CONVERSATION +THE sluggish day began to break. Even its position +terrestrially is one of the elements of a new interest, +and for no particular reason save that the incident of +the night had occurred there, Oak went again into +the plantation. Lingering and musing here, he heard +the steps of a horse at the foot of the hill, and soon +there appeared in view an auburn pony with a girl on +its back, ascending by the path leading past the cattle+ +shed. She was the young woman of the night before. +Gabriel instantly thought of the hat she had mentioned +as having lost in the wind; possibly she had come to +look for it. He hastily scanned the ditch and after +walking about ten yards along it, found the hat among the +leaves. Gabriel took it in his hand and returned to his +hut. Here he ensconced himself, and peeped through +the loophole in the direction of the riders approach. +She came up and looked around -- then on the other +side of the hedge. Gabriel was about to advance and +restore the missing article when an unexpected per+ +formance induced him to suspend the action for the +present. The path, after passing the cowshed, bisected +the plantation. It was not a bridle-path -- merely a +pedestrian's track, and the boughs spread horizontally +at a height not greater than seven feet above the ground, +which made it impossible to ride erect beneath them. +The girl, who wore no riding-habit, looked around for +a moment, as if to assure herself that all humanity was +out of view, then dexterously dropped backwards flat +upon the pony's back, her head over its tail, her feet +against its shoulders, and her eyes to the sky. The +rapidity of her glide into this position was that of a +kingfisher -- its noiselessness that of a hawk. Gabriel's +eyes had scarcely been able to follow her. The tall lank +pony seemed used to such doings, and ambled +

+along unconcerned. Thus she passed under the level boughs. +The performer seemed quite at home anywhere +between a horse's head and its tail, and the necessity +for this abnormal attitude having ceased with the +passage of the plantation, she began to adopt another, +even more obviously convenient than the first. She had +no side-saddle, and it was very apparent that a firm +seat upon the smooth leather beneath her was un+ +attainable sideways. Springing to her accustomed +perpendicular like a bowed sapling, and satisfying her, +self that nobody was in sight, she seated herself in the +manner demanded by the saddle, though hardly expected +of the woman, and trotted off in the direction of Tewnell +Mill. +Oak was amused, perhaps a little astonished, and +hanging up the hat in his hut, went again among his +ewes. An hour passed, the girl returned, properly +seated now, with a bag of bran in front of her. On +nearing the cattle-shed she was met by a boy bringing +a milking-pail, who held the reins of the pony whilst +she slid off. The boy led away the horse, leaving the +pail with the young woman. +Soon soft spirts alternating with loud spirts came +in regular succession from within the shed, the obvious +sounds of a person milking a cow. Gabriel took the +lost hat in his hand, and waited beside the path she +would follow in leaving the hill. +She came, the pail in one hand, hanging against her +knee. The left arm was extended as a balance, enough +of it being shown bare to make Oak wish that the event +ha happened in the summer, when the whole would +have been revealed. There was a bright air and manner +about her now, by which she seemed to imply that the +desirability of her existence could not be questioned; +and this rather saucy assumption failed in being offensive, +because a beholder felt it to be, upon the whole, true. +Like exceptional emphasis in the tone of a genius, that +which would have made mediocrity ridiculous was an +addition to recognised power. It was with some +surprise that she saw Gabriel's face rising like the +moon behind the hedge. +The adjustment of the farmer's hazy conceptions of +her +

+charms to the portrait of herself she now presented +him with was less a diminuition than a difference. The +starting-point selected by the judgment was. her height +She seemed tall, but the pail was a small one, and the +hedge diminutive; hence, making allowance for error +by comparison with these, she could have been not +above the height to be chosen by women as best. All +features of consequence were severe and regular. It +may have been observed by persons who go about the +shires with eyes for beauty, that in Englishwoman a +classically-formed face is seldom found to be united +with a figure of the same pattern, the highly-finished +features being generally too large for the remainder of +the frame ; that a graceful and proportionate figure of +eight heads usually goes off into random facial curves. +Without throwing a Nymphean tissue over a milkmaid, +let it be said that here criticism checked itself as out +of place, and looked at her proportions with a long +consciousness of pleasure. From the contours of her +figure in its upper part, she must have had a beautiful +neek and shoulders ; but since her infancy nobody had +ever seen them. Had she been put into a low dress +she would have run and thrust her head into a bush. +Yet she was not a shy girl by any means; it was merely +her instinct to draw the line dividing the seen from the +unseen higher than they do it in towns. +That the girl's thoughts hovered about her face +and form as soon as she caught Oak's eyes conning the +same page was natural, and almost certain. The self+ +consciousness shown would have been vanity if a little +more pronounced, dignity if a little less. Rays of male +vision seem to have a tickling effect upon virgin faces +in rural districts ; she brushed hers with her hand, as if +Gabriel had been irritating its pink surface by actual +touch, and the free air of her previous movements was +reduced at the same time to a chastened phase of +itself. Yet it was the man who blushed, the maid not +at all. +" I found a hat,' said Oak. +" It is mine,' said she, and, from a sense of proportion, +kept down to a small smile an inclination to laugh dis+ +tinctly : "it flew away last night.' +" One o'clock this morning ? ' +

+" Well -- it was.' She was surprised. " How did you +know ? ' she said. +" I was here.' +" You are Farmer Oak, are you not ? ' +" That or thereabouts. I'm lately come to this place.' +" A large farm ? ' she inquired, casting her eyes round, +and swinging back her hair, which was black in the +shaded hollows of its mass; but it being now an hour +past sunrise, the rays touched its prominent curves with +a colour of their own. +" No ; not large. About a hundred.' (In speaking +of farms the word "acres ' is omitted by the natives, by +analogy to such old expressions as "a stag of ten.') +' "I wanted my hat this morning,' she went on. +had to ride to Tewnell Mill.' +"Yes you had.' +"How do you know?' +"I saw you! +"Where?' she inquired, a misgiving bringing every +muscle of her lineaments and frame to a standstill. +"Here-going through the plantation, and all down +the hill,' said Farmer Oak, with an aspect excessively +knowing with regard to some matter in his mind, as he +gazed at a remote point in the direction named, and then +turned back to meet his colloquist's eyes. +A perception caused him to withdraw his own eyes +from hers as suddenly as if he had been caught in a +theft. Recollection of the strange antics she had +indulged in when passing through the trees, was suc+ +ceeded in the girl by a nettled palpitation, and that' by +a hot face. It was a time to see a woman redden who +was not given to reddening s a rule; not a point in +the milkmaid but was of the deepest rose-colour. From +the Maiden's Blush, through all varieties of the Provence +down to the Crimson Tuscany, the countenance of Oak's +acquaintance quickly graduated ; whereupon he, in con+ +siderateness, turned away his head. +The sympathetic man still looked the other way, and +wondered when she would recover coolness sufficient to +justify him in facing her again. He heard what seemed +to be the flitting of a +

+dead leaf upon the breeze, and +looked. She had gone away. +With an air between that of Tragedy and Comedy ! +Gabriel returned to his work. +Five mornings and evenings passed. The young +woman came regularly to milk the healthy cow or to +attend to the sick one, but never allowed her vision to +stray in the direction of Oak's person. His want of +tact had deeply offended her -- not by seeing what he +could not help, but by letting her know that he had +seen it. For, as without law there is no sin, without +eyes there is no indecorum; and she appeared to feel +that Gabriel's espial had made her an indecorous woman +without her own connivance. It was food for great regret +with him; it was also a contretemps which touched into +life a latent heat he had experienced in that direction. +The acquaintanceship might, however, have ended in +a slow forgetting, but for an incident which occurred at +the end of the same week. One afternoon it began to +freeze, and the frost increased with evening, which drew +on like a stealthy tightening of bonds. It was a time +when in cottages the breath of the sleepers freezes to +the sheets; when round the drawing-room fire of a +thick-walled mansion the sitters' backs are cold, even +whilst their faces are all aglow. Many a small bird went +to bed supperless that night among the bare boughs. +As the milking-hour drew near, Oak kept his usual +watch upon the cowshed. At last he felt cold, and +shaking an extra quantity of bedding round the yeaning +ewes he entered the hut and heaped more fuel upon +the stove. The wind came in at the bottom of the door, +and to prevent it Oak laid a sack there and wheeled the +cot round a little more to the south. Then the wind +spouted in at a ventilating hole -- of which there was one +on each side of the hut. +Gabriel had always known that when the fire was +lighted and the door closed one of these must be kept +open -- that chosen being always on the side away from +the wind. Closing the slide to windward, he turned to +open the other; on second -- -thoughts the farmer con+ +sidered that he would first sit down leaving both +closed for a minute or two, till the temperature of the +hut was a little raised. He sat down. +

+His head began to ache in an unwonted manner, and, +fancying himself weary by reason of the broken rests of +the preceding nights, Oak decided to get up, open the +slide, and then allow himself to fall asleep. He fell +asleep, however, without having performed the necessary +preliminary. +How long he remained unconseious Gabriel never +knew. During the first stages of his return to percep+ +tion peculiar deeds seemed to be in course of enactment. +His dog was howling, his head was aching fearfully -- +somebody was pulling him about, hands were loosening +his neckerchief. +On opening his eyes he found that evening had sunk +to dusk in a strange manner of unexpectedness. The +young girl with the remarkably pleasant lips and white +teeth was beside him. More than this -- astonishingly +more -- his head was upon her lap, his face and neck +were disagreeably wet, and her fingers were unbuttoning +his collar. +"Whatever is the matter?' said Oak, vacantly. +She seemed to experience mirth, but of too insignifi+ +cant a kind to start enjoyment. +"Nothing now', she answered, "since you are not +dead It is a wonder you were not,suffocated in this +hut of yours.' +"Ah, the hut ! ' murmured Gabriel. "I gave ten +pounds for that hut. But I'll sell it, and sit under +thatched hurdles as they did in old times, curl up +to sleep in a lock of straw! It played me nearly the +same trick the other day .! ' Gabriel, by way of emphasis, +brought down his fist upon the floor. +"It was not exactly the fault of the hut,' she ob+ +served in a tone which showed her to be that novelty +among women -- one who finished a thought before +beginning the sentence which was to convey it. " You +should I think, have considered, and not have been so +foolish as to leave the slides closed.' +"Yes I suppose I should,' said Oak, absently. He +was endeavouring to catch and appreciate the sensation +of being thus with her, his head upon her dress, before +the event passed on into the heap of bygone things. +He wished she knew his impressions ; but he would as +soon have thought of carrying an odour in a net as of +attempting to convey the intangibilities +

+of his feeling +in the coarse meshes of language. So he remained +silent. +She made him sit up, and then Oak began wiping +his face and shaking himself like a Samson. "How +can I thank 'ee ? ' he said at last, gratefully, some of the +natural rusty red having returned to his face. + " Oh, never mind that,' said the girl, smiling, and +allowing her smile to hold good for Gabriel's next +remark, whatever that might prove to be. +"How did you find me?" +"I heard your dog howling and scratching at the +door of the hut when I came to the milking (it was so +lucky, Daisy's milking is almost over for the season, and + I shall not come here after this week or the next). The +dog saw me, and jumped over to me, and laid hold of +my skirt. I came across and looked round the hut the +very first thing to see if the slides were closed. My +uncle has a hut like this one, and I have heard him tell +his shepherd not to go to sleep without leaving a slide +open. I opened the door, and there you were like +dead. I threw the milk over you, as there was no +water, forgetting it was warm, and no use.' +"I wonder if I should have died ? ' Gabriel said, in a +low voice, which was rather meant to travel back to +himself than to her. +"O no," the girl replied. She seemed to prefer a +less tragic probability ; to have saved a man from death +'involved talk that should harmonise with the dignity of +such a deed -- and she shunned it. +"I believe you saved my life, Miss -- -- I don!t know +your name. I know your aunt's, but not yours.' +" I would just as soon not tell it -- rather not. There +is no reason either why I should, as you probably will +never have much to do with me.' + " Still, I should like to know.' +" You can inquire at my aunt's -- she will tell you.' +'My name is Gabriel Oak.' +"And mine isn't. You seem fond of yours in +speaking it so decisively, Gabriel Oak.' +

+" You see, it is the only one I shall ever have, and I +must make the most of it.' +" I always think mine sounds odd and disagreeable.' +"I should think you might soon get a new one.' +"Mercy ! -- how many opinions you keep about you +concerning other people, Gabriel Oak. +"Well Miss-excuse the words-I thought you +would like them But I can't match you I know in +napping out my mind upon my tongue. I never was +very clever in my inside. But I thank you. Come +give me your hand!' +She hesitated, somewhat disconcerted at Oak's old+ +fashioned earnest conclusion. to a dialogue lightly +carried on."Very well,' she said, and gave him her +hand, compressing her lips to a demure impassivity. +He held it but an instant, and in his fear of being too +demonstrative, swerved to the opposite extreme, touching +her fingers with the lightness of a small-hearted person. +" I am sorry,' he said, the instant after. +" What for?' +"You may have it again if you like; there it is.' +She gave him her hand again. +Oak held it longer this time -- indeed, curiously long. +"How soft it is -- being winter time, too -- not chapped +or rough or anything!' he said. +"There -- that's long enough,' said she, though with+ +out pulling it away "But I suppose you are thinking +you would like to kiss it? You may if you want to.' +"I wasn't thinking of any such thing,' said Gabriel, +simply ; "but I will' +"That you won't!' She snatched back her hand. +Gabriel felt himself guilty of another want of tact. +"Now find out my name,' she said, teasingly; and +withdrew. + +

+GABRIEL'S RESOLVE -- THE VISIT -- THE MISTAKE +THE only superiority in women that is tolerable to the +rival sex is, as a rule, that of the unconscious kind ; but +a superiority which recognizes itself may sometimes +please by suggesting possibilities of capture to the +subordinated man. +This well-favoured and comely girl soon made appre+ +ciable inroads upon the emotional constitution of young +Farmer Oak. +Love, being an extremely exacting usurer (a sense of +exorbitant profit, spiritually, by an exchange of hearts, +being at the bottom of pure passions, as that of exorbi+ +tant profit, bodily or materially, is at the bottom of +those of lower atmosphere), every morning Oak's feelings +were as sensitive as the money-market in calculations +upon his chances. His dog waited for his meals in a +way so like that in which Oak waited for the girl's +presence, that the farmer was quite struck with the +resemblance, felt it lowering, and would not look at the +dog. However, he continued to watch through the +hedge for her regular coming, and thus his sentiments +towards her were ideepened without any corresponding +effect being produced upon herself. Oak had nothing +finished and ready to say as yet, and not being able +to frame love phrases which end where they begin ; +passionate tales -- + + -- -Full of sound and fury + -- -signifting nothing -- + +he said no word at all. +By making inquiries he found that the girl's name +was Bathsheba Everdene, and that the cow would go +dry in about seven days. He dreaded the eight day. +At last the eighth day came. The cow had ceased +to give milk for that year, and Bathsheba Everdene +came up the hill no more. Gabriel had reached a +pitch of existence he never +

+could have anticipated a +short time before. He liked saying 'Bathsheba' as a +private enjoyment instead of whistling; turned over his +taste to black hair, though he had sworn by brown ever +since he was a boy, isolated himself till the space he +filled in a possible strength in an actual weakness. Marriage +transforms a distraction into a support, the power of +which should be, and happily often is, in direct pro+ +portion to the degree of imbecility it supplants. Oak +began now to see light in this direction, and said to +himself, "I'll make her my wife, or upon my soul I shall +be good for nothing .! ' +All this while he was perplexing himself about an +errand on which he might consistently visit the cottage +of Bathsheba's aunt. +He found his opportunity in the death of a ewe, +mother of a living lamb. On a day which had a +summer face and a winter constitution-a fine January +morning, when there was just enough blue sky visible to +make cheerfully-disposed people wish for more, and an +occasional gleam of silvery sunshine, Oak put the lamb +into a respectable Sunday basket, and stalked across the +fields to the house of Mrs. Hurst, the aunt -- George, +the dog walking behind, with a countenance of great +concern at the serious turn pastoral affairs seemed to be +taking. +Gabriel had watched the blue wood-smoke curling +from the chimney with strange meditation. At evening +he had fancifully traced it down the chimney to the +spot of its origin -- seen the hearth and Bathsheba +beside it -- beside it in her out-door dress; for the +clothes she had worn on the hill were by association +equally with her person included in the compass of his +affection; they seemed at this early time of his love a +necessary ingredient of the sweet mixture called Bath+ +sheba Everdene. +He had made a toilet of a nicely-adjusted kind -- of a +nature between the carefully neat and the carelessly +ornate -- of a degree between fine-market-day and wet+ +Sunday selection. He thoroughly cleaned his silver +watch-chain with whiting, put new lacing straps to his +boots, looked to the brass eyelet-holes, +

+ went to the +inmost heart of the plantation for a new walking-stick, +and trimmed it vigorously on his way back; took a new +handkerchief from the bottom of his clothes-box, put +on the light waistcoat patterned all over with sprigs +of an elegant flower uniting the beauties of both rose +and lily without the defects of either, and used all the +hair-oil he possessed upon his usually dry, sandy, and +inextricably curly hair, till he had deepened it to a +splendidly novel colour, between that of guano and +Roman cement, making it stick to his head like mace +round a nutmeg, or wet seaweed round a boulder after +the ebb. +Nothing disturbed the stillness of the cottage save + the chatter of a knot of sparrows on the eaves; one +might fancy scandal and rumour to be no less the +staple topic of these little coteries on roofs than of +those under them. It seemed that the omen was an +unpropitious one, for, as the rather untoward commence+ +ment of Oak's overtures, just as he arrived by the garden +gate, he saw a cat inside, going into various arched shapes +and fiendish convulsions at the sight of his dog George. +The dog took no notice , for he had arrived at an age +at which all superfluous barking was cynically avoided +as a waste of breath -- -in fact he never barked even +at the sheep except to order, when it was done with +an absolutely neutral countenance, as a sort of Com+ +mination-service, which, though offensive, had to be +gone through once now and then to frighten the flock +for their own good. +A voice came from behind some laurel-bushes into +which the cat had run: +"Poor dear! Did a nasty brute of a dog want to +kill it; -- did he poor dear !' +"I beg your pardon,' said Oak to the voice, 'but +George was walking on behind me with a temper as +mild as milk.' +Almost before he had ceased speaking, Oak was +seized with a misgiving as to whose ear was the recipient +of his answer. Nobody appeared, and he heard the +person retreat among the bushes. +Gabriel meditated, and so deeply that he brought +small furrows into his forehead by sheer force of +reverie. Where the +

+issue of an interview is as likely +to be a vast change for the worse as for the better, +any initial difference from expectation causes nipping +sensations of failure. Oak went up to the door a little +abashed : his mental rehearsal and the reality had had +no common grounds of opening. +Bathsheba's aunt was indoors. " Will you tell Miss +Everdene that somebody would be glad to speak to +her ?'said Mr. Oak. (Calling one's self merely Some+ +body, without giving a name, is not to be taken as +an example of the ill-breeding of the rural world: it +springs from a refined modesty, of which townspeople, +with their cards and announcements, have no notion +whatever.) +Bathsheba was out. The voice had evidently been +hers. +" Will you come in, Mr. Oak ? ' +"Oh, thank 'ee, said Gabriel, following her to the +fireplace. "I've brought a lamb for Miss Everdene. +I thought she might like one to rear; girls do.' +" She might,' said Mrs. Hurst, musingly ; " though +she's only a visitor here. If you will wait a minute, +Bathsheba will be in.' +" Yes, I will wait,' said Gabriel, sitting down. " The +lamb isn't really the business I came about, Mrs. Hurst. +In short, I was going to ask her if she'd like to be +married.' +"And were you indeed ?' +" Yes. Because if she would, I should be very glad +to marry her. D'ye know if she's got any other young +man hanging about her at all ?' +"Let me think," said Mrs. Hurst, poking the fire +superfluously.... " Yes -- bless you, ever so many young +men. You see, Farmer Oak, she's so good-looking, and +an excellent scholar besides -- she was going to be a +governess once, you know, only she was too wild. Not +that her young men ever come here -- but, Lord, in the +nature of women, she must have a dozen ! ' +" That's unfortunate,' said Farmer Oak, contemplating +a crack in the stone floor with sorrow. "I'm only an +every-day sort of man, and my only chance was in being +the first comer... , Well, there's no use in my waiting, +for that was all I came about: so I'll take myself off +home-along, Mrs. Hurst.' +When Gabriel had gone about two hundred yards +along the +

+down, he heard a "hoi-hoi .! " uttered behind +him, in a piping note of more treble quality than that +in which the exclamation usually embodies itself when +shouted across a field. He looked round, and saw a girl +racing after him, waving a white handkerchief. +Oak stood still -- and the runner drew nearer. It was +Bathsheba Everdene. Gabriel's colour deepened: hers +was already deep, not, as it appeared, from emotion, +but from running. +"Farmer Oak -- I -- ' she said, pausing for want of +breath pulling up in front of him with a slanted face +and putting her hand to her side. +"I have just called to see you ' said Gabriel, pending +her further speech. +"Yes-I know that,! she said panting like a robin, +her face red and moist from her exertions, like a peony +petal before the sun dries off the dew. "I didn't know +you had come to ask to have me, or I should have come +in from the garden instantly. I ran after you to say -- +that my aunt made a mistake in sending you away from +courting me -- -- -- ' +Gabriel expanded."I'm sorry to have made you +run so fast, my dear,' he said, with a grateful sense of +favours to come. "Wait a bit till you've found your +breath.' +" -- It was quite a mistake-aunt's telling you I had +a young man "already,'- Bathsheba went on. " I haven't +a sweetheart at all -- and I never had one, and I thought +that, as times go with women, it was such a pity to send +you away thinking that I had several.' +"Really and truly I am glad to hear that.!' said .= +Farmer Oak, smiling one of his long special smiles, and +blushing with gladness. He held out his hand to take +hers, which, when she had eased her side by pressing +it there, was prettily extended upon her bosom to still +her loud-beating heart. Directly he seized it she put +it behind her, so that it slipped through his fingers like +an eel. " +"I have a nice snug little farm,' said Gabriel, with +half a degree less assurance than when he had seized +her hand. +"Yes ; you have.' +"A man has advanced me money to begin with, but +still, it +

+will soon be paid off and though I am only an +every-day sort of man, I have got on a little since I was +a boy.' Gabriel uttered "a little' in a tone to-show +her that it was the complacent form of "a great deal.' +He continued : " When we be married, I am quite sure +I can work twice as hard as I do now.' + He went forward and stretched out his arm again. +Bathsheba had overtaken him at a point beside which +stood a low stunted holly bush, now laden with red +berries. Seeing his advance take the form of an attitude +threatening a possible enclosure, if not compression, of +her person, she edged off round the bush. +" Why, Farmer Oak,' she said, over the top, looking +at him with rounded eyes, "I never said I was going to +marry you.' +" Well -- that is a tale .! ' said Oak, with dismay. " To +run after anybody like this, and then say you don"t +want him ! ' +"What I meant to tell you was only this,' she said +eagerly, and yet half conscious of the absurdity of the +position she had made for herself -- "that nobody has +got me yet as a sweetheart, instead of my having a +dozen, as my aunt said; I hate to be thought men's +property in that way, though possibly I shall be had +some day. Why, if I'd wanted you I shouldn't have +run after you like this ; 'twould have'been the forwardest +thing ! But there was no harm in 'hurrying to correct +a piece of false news that had been told you.' +"Oh, no -- no harm at all." But there is such a thing +as being too generous in expressing a judgment impuls+ +ively, and Oak added with a more appreciative sense +of all the circumstances -- ' Well, I am not quite certain +it was no harm.' +"Indeed, I hadn't time to think before starting +whether I wanted to marry or not, for you'd have been +gone over the hill.' +" Come,' said Gabriel, freshening again ; "think a +minute or two. I'll wait a while, Miss Everdene. Will +you marry me? Do, Bathsheba. I love you far more +than common!' +"I'll try to think,' she observed, rather more timor+ +ously ; "if I can think out of doors; my mind spreads +away so.' +"But you can give a guess.' +

+"Then give me time.' Bathsheba looked thought+ +fully into the distance, away from the direction in which +Gabriel stood. +"I can make you happy,' said he to the back of her +head, across the bush. "You shallo have as piano in a +year or two -- -farmers' wives are getting to have pianos +now -- and I'll practise up the flute right well to play +with you in the evenings.' +" Yes ; I should like that.' +"And have one of those little ten-pound" gigs for +market -- and nice flowers, and birds -- cocks and hens +I mean, because they be useful,' continued Gabriel, +feeling balanced between poetry and practicality. +"I should like it very much.' +"And a frame for cucumbers -- like a gentlman and +lady.' +"Yes.' +"And when the wedding was over, we'd have it put +in the newspaper list of marriages.' +" Dearly I should like that ! ' +"And the babies in the births -- every man jack of +'em! And at home by the fire, whenever you look up, +there I shall be -- and whenever I look up' there will +be you.' +"Wait wait and don't be improper .!' +Her countenance fell, and she was silent awhile. +He regarded the red berries between them over and +over again, to such an extent, that holly seemed in +his after life to be a cypher signifying a proposal of +marriage. Bathsheba decisively turned to him. +"No;' 'tis no use,' she said. 'I don't want to marry +you. ' +' Try.' +"I have tried hard all the time I've been thinking; +for a marriage would be very nice in one sense. +People would talk about me, and think I had won my +battle, and I should feel triumphant, and' all that, +But a husband -- -- ' - + +" Well .! ' +" Why, he'd always be there, as you say; whenever +I looked up, there he'd be.' +" Of course he would -- I, that is.' +

+" Well, what I mean is that I shouldn't mind being +a bride at a wedding, if I could be one without having +a husband. But since a woman can't show off in that +way by herself, I shan't marry -- at least yet.' +' That's a terrible wooden story.' +At this criticism of her statement Bathsheba made +an addition to her dignity by a slight sweep away +from him. +"Upon my heart and soul, I don't know what a +maid can say stupider than that,' said Oak. "But +dearest,' he continued in a palliative voice, "don't be +like it !.' Oak sighed a deep honest sigh -- none the +less so in that, being like the sigh of a pine plantation, +it was rather noticeable as a disturbance of the atmo+ +sphere. " Why won't you have me ? ' he appealed, +creeping round the holly to reach her side. +" I cannot,' she said, retreating. +"But why ?' he persisted, standing still at last in +despair of ever reaching her, and facing over the +bush. +' Because I don't love you.' +" Yes, but -- -- ' +She contracted a yawn to an inoffensive smallness, +so that it was hardly ill-mannered at all. "I don't love +you,' she said.' +"But I love you -- and, as for myself, I am content +to be liked.' +" O Mr. Oak -- that's very fine ! You'd get to +despise me.' +"Never,' said Mr Oak, so earnestly that he seemed +to be coming, by the forceof his words, straight +through the bush and into her arms. "I shall do one +thing in this life -- one thing certain -- that is, love you, +and long for you, and keep wanting you till I die.' His +voice had a genuine pathos now, and his large brown +hands perceptibly trembled. +"It seems dreadfully wrong not to have you when +you feel so much!' she said with a little distress, and +looking hopeleely around for some means of escape +from her moral dilemma. " H(ow I wish I hadn't run +after you!' However she seemed to have a short cut +for getting back to cheerfulness, and set her face to +signify archness. "It wouldn't do, Mr Oak. I want +somebody to tame me; I am too independent ; and +you would never be able to, I know.' +

+Oak cast his eyes down the field in a way implying +that it was useless to attempt argument. +" Mr. Oak,' she said, with luminous distinctness and +common sense, " you are better off than I. I have +hardly a penny in the world -- I am staying with my +aunt for my bare sustenance. I am better educated +than you -- and I don't love you a bit: that's my side +of the case. Now yours: you are a farmer just begin+ +ing; and you ought in common prudence, if you marry +at all (which you should certainly not think of doing +at present) to marry a woman with money, who would +admiration. +"That's the very thing I had been thinking myself !' +he naively said. +Farmer Oak had one-and-a-half Christian character +istics too many to succeed with Bathsheba : his humility, +and a superfluous moiety of honesty. Bathsheba was +decidedly disconcerted, +"Well, then, why did you come and disturb me?' +she said, almost angrily, if not quite, an enlarging red +spot rising in each cheek. +" I can't do what I think would be -- would be -- -- ' +" Right ? ' +" No : wise.' +" You have made an admission now, Mr. Oak,' she +exclaimed, with even more hauteur, and rocking her +head disdainfully. 'After that, do you think I could +marry you? Not if I know it.' +He broke in passionately ! "But don't mistake me +like that! Because I am open enough to own what +every man in my shoes would have thought of, you +make your colours come up your face, and get crabbed +with me. That about your not being good enough for +me is nonsense. You speak like a lady -- all the parish +notice it, and your uncle at Weatherbury is, I have +heerd, a large farmer -- much larger than ever I shall +be. May I call in the evening, or will you walk along +with me o' Sundays? I don't want you to make-up +your mind at once, if you'd rather not.' +

+" No -- no -- I cannot. Don't press me any more -- +don't. I don't love you -- so 'twould be ridiculous,' +she said, with a laugh. +No man likes to see his emotions the sport of a +merry-go-round of skittishness. " Very well,' said Oak, +firmly, with the bearing of one who was going to give ' +his days and nights to Ecclesiastes for ever. "Then +I'll ask you no more.' + +

+DEPARTURE OF BATHSHEBA -- A PASTORAL TRAGEDY +THE news which one day reached Gabriel, that Bath+ +sheba Everdene had left the neighbourhood, had an +influence upon him which might have surprised any +who never suspected that the more emphatic the renun+ +ciation the less absolute its character. +It may have been observed that there is no regula +path for getting out of love as there is for getting in. +Some people look upon marriage as a short cut that way, +but it has been known to fail. Separation, which was +the means that chance offered to Gabriel Oak by +Bathsheba's disappearance though effectual with people +of certain humours is apt to idealise the removcd object +with others -- notably those whose affection, placid and +regular as it may be flows deep and long. Oak belonged +to the even-tempered order of humanity, and felt the +secret fusion of himself in Bathsheba to be burning with +a finer flame now that she was gone -- that was all. +His incipient friendship with her aunt-had been +nipped by the failure of his suit, and all that Oak learnt +of Bathsheba's movements was done indirectly. It ap+ +peared that she had gone to a place called Weatherbury, +more than twenty miles off, but in what capacity -- +whether as a visitor, or permanently, he could not +discover. +Gabriel had two dogs. George, the elder, exhibited +an ebony-tipped nose, surrounded by a narrow margin +of pink flesh, and a coat marked in random splotches +approximating in colour to white and slaty grey ; but the +grey, after years of sun and rain, had been scorched and +washed out of the more prominent locks, leaving them +of a reddish-brown, as if the blue component of the grey +had faded, like the indigo from the same kind of colour in +Turner's pictures. In substance it had originally been +hair, but long contact with sheep seemed +

+to be turning +it by degrees into wool of a poor quality and staple. +This dog had originally belonged to a shepherd of +inferior morals and dreadful temper, and the result was +that George knew the exact degrees of condemnation +signified by cursing and swearing of all descriptions +better than the wickedest old man in the neighbourhood. +Long experience had so precisely taught the animal the +difference between such exclamations as 'Come in .! ' +and 'D -- -- ye, come in !.' that he knew to a hair's +breadth the rate of trotting back from the ewes' tails +that each call involved, if a staggerer with the sheep +crook was to be escaped. Though old, he was clever +and trustworthy still. +The young dog, George's son, might possibly have +been the image of his mother, for there was not much +resemblance between him and George. He was learn+ +ing the sheep-keeping business, so as to follow on at +the flock when the other should die, but had got no +further than the rudiments as yet -- still finding an +insuperable difculty in distinguishing between doing a +thing well enough and doing it too well. So earnest +and yet so wrong-headed was this young dog (he had no, +name in particular, and answered with perfect readiness +to any pleasant interjection), that if sent behind the +flock to help them on, he did it so thoroughly that he +would have chased them across the whole county with +the greatest pleasure if not called off or reminded when +to step by the example of old George. +Thus much for the dogs. On the further side of +Norcombe Hill was a chalk-pit, from which chalk had +been drawn for generations, and spread over adjacent +farms. Two hcdges converged upon it in the form of +a V, but without quite meeting. The narrow opening +left, which was immediately over the brow of the pit, +was protected by a rough railing. +One night, when Farmer Oak had returned to, his +house, believing there would be no further necessity for +his attendance on the down, he called as usual to the +dogs, previously to shutting them up in the outhouse till +next morning. Only one responded -- old George ; the +other-could not be found, either in the house, lane, or +garden. - Gabriel then remembered +

+that he had left the +two dogs on the hill eating a dead lamb (a kind of meat +he usually kept from them, except when other food-ran +finished his meal, he went indoors to the luxury of a bed, +which latterly he had only enjoyed on Sundays. +It was a still, moist night. Just before dawn he was +assisted in waking by the abnormal reverberation of +familiar music. To the shepherd, the note of the sheep' +chronic sound that only makes itself noticed by ceasing +ever distant, that all is well in the fold. In the solemn +This exceptional ringing may be caused in two ways -- + +by the rapid feeding of the sheep \ No newline at end of file diff --git a/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/geo.binary b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/geo.binary new file mode 100644 index 0000000000000..7a2270a241f5f Binary files /dev/null and b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/geo.binary differ diff --git a/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/pic.binary b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/pic.binary new file mode 100644 index 0000000000000..d335ed1297224 Binary files /dev/null and b/libs/lz4/src/test/resources/org/elasticsearch/lz4/calgary/pic.binary differ diff --git a/server/build.gradle b/server/build.gradle index 02e70b9b8346e..11852bad24a39 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -69,6 +69,7 @@ dependencies { api project(':libs:elasticsearch-secure-sm') api project(':libs:elasticsearch-x-content') api project(":libs:elasticsearch-geo") + api project(":libs:elasticsearch-lz4") implementation project(':libs:elasticsearch-plugin-classloader') @@ -92,9 +93,6 @@ dependencies { api project(":libs:elasticsearch-cli") api 'com.carrotsearch:hppc:0.8.1' - // LZ4 - api 'org.lz4:lz4-java:1.8.0' - // time handling, remove with java 8 time api "joda-time:joda-time:${versions.joda}" @@ -289,11 +287,6 @@ tasks.named("thirdPartyAudit").configure { if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) { ignoreMissingClasses 'javax.xml.bind.DatatypeConverter' } - - ignoreViolations( - // from java-lz4 - 'net.jpountz.util.UnsafeUtils' - ) } tasks.named("dependencyLicenses").configure { diff --git a/server/src/main/java/org/elasticsearch/transport/Compression.java b/server/src/main/java/org/elasticsearch/transport/Compression.java index cb8fc010de366..f4a2173da7b83 100644 --- a/server/src/main/java/org/elasticsearch/transport/Compression.java +++ b/server/src/main/java/org/elasticsearch/transport/Compression.java @@ -8,10 +8,16 @@ package org.elasticsearch.transport; +import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.lz4.ESLZ4Compressor; +import org.elasticsearch.lz4.ESLZ4Decompressor; import java.io.IOException; import java.io.OutputStream; @@ -27,6 +33,7 @@ public enum Scheme { private static final byte[] DEFLATE_HEADER = new byte[]{'D', 'F', 'L', '\0'}; private static final byte[] LZ4_HEADER = new byte[]{'L', 'Z', '4', '\0'}; private static final int LZ4_BLOCK_SIZE; + private static final boolean USE_FORKED_LZ4; static { String blockSizeString = System.getProperty("es.transport.compression.lz4_block_size"); @@ -39,6 +46,8 @@ public enum Scheme { } else { LZ4_BLOCK_SIZE = 64 * 1024; } + + USE_FORKED_LZ4 = Booleans.parseBoolean(System.getProperty("es.compression.use_forked_lz4", "true")); } public static boolean isDeflate(BytesReference bytes) { @@ -68,9 +77,23 @@ private static boolean validateHeader(BytesReference bytes, byte[] header) { return true; } + public static LZ4FastDecompressor lz4Decompressor() { + if (USE_FORKED_LZ4) { + return ESLZ4Decompressor.INSTANCE; + } else { + return LZ4Factory.safeInstance().fastDecompressor(); + } + } + public static OutputStream lz4OutputStream(OutputStream outputStream) throws IOException { outputStream.write(LZ4_HEADER); - return new ReuseBuffersLZ4BlockOutputStream(outputStream, LZ4_BLOCK_SIZE, LZ4Factory.safeInstance().fastCompressor()); + LZ4Compressor lz4Compressor; + if (USE_FORKED_LZ4) { + lz4Compressor = ESLZ4Compressor.INSTANCE; + } else { + lz4Compressor = LZ4Factory.safeInstance().fastCompressor(); + } + return new ReuseBuffersLZ4BlockOutputStream(outputStream, LZ4_BLOCK_SIZE, lz4Compressor); } } @@ -79,5 +102,4 @@ public enum Enabled { INDEXING_DATA, FALSE } - } diff --git a/server/src/main/java/org/elasticsearch/transport/Lz4TransportDecompressor.java b/server/src/main/java/org/elasticsearch/transport/Lz4TransportDecompressor.java index 316413a0e01f7..23c3eaef6398f 100644 --- a/server/src/main/java/org/elasticsearch/transport/Lz4TransportDecompressor.java +++ b/server/src/main/java/org/elasticsearch/transport/Lz4TransportDecompressor.java @@ -24,7 +24,6 @@ package org.elasticsearch.transport; import net.jpountz.lz4.LZ4Exception; -import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import org.apache.lucene.util.BytesRef; @@ -120,7 +119,7 @@ private enum State { private boolean hasSkippedESHeader = false; public Lz4TransportDecompressor(PageCacheRecycler recycler) { - this.decompressor = LZ4Factory.safeInstance().fastDecompressor(); + this.decompressor = Compression.Scheme.lz4Decompressor(); this.recycler = recycler; this.pages = new ArrayDeque<>(4); }