From b6e4816dd8c1946f8b7648e9ac6c5d3b5d8d6c51 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 23 Feb 2021 11:54:15 +0100 Subject: [PATCH] Always read footer checksum from FileInfo (#69415) Searchable snapshots IndexInput implementations detect read operations that are executed on the last 16 bytes of files which contain the footer checksum, and then serves them from memory (FileInfo) instead of relying on cache mechanisms. But the current implementation expect an exact checksum read: ie a read operation that starts at file length - 16 bytes and expects to read exactly 16 bytes. This is not always the case as IndexInput implementations extend BufferedIndexInput which uses an internal buffer of 1024 bytes: in some case the remaining length (ie, the last portion of the file) might be less than 16 bytes long and thus a very small read is engaged using cache (or direct) read where it could instead be also served from memory. --- .../index/store/BaseSearchableSnapshotIndexInput.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java index 52edf9c004359..701d264b332f9 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java @@ -95,11 +95,11 @@ protected final void readInternal(ByteBuffer b) throws IOException { */ private boolean maybeReadChecksumFromFileInfo(ByteBuffer b) throws IOException { final int remaining = b.remaining(); - if (remaining != CodecUtil.footerLength()) { + if (remaining > CodecUtil.footerLength()) { return false; } final long position = getFilePointer() + this.offset; - if (position != fileInfo.length() - CodecUtil.footerLength()) { + if (position < fileInfo.length() - CodecUtil.footerLength()) { return false; } if (isClone) { @@ -107,7 +107,8 @@ private boolean maybeReadChecksumFromFileInfo(ByteBuffer b) throws IOException { } boolean success = false; try { - b.put(checksumToBytesArray(fileInfo.checksum())); + final byte[] checksum = checksumToBytesArray(fileInfo.checksum()); + b.put(checksum, CodecUtil.footerLength() - remaining, remaining); success = true; } catch (NumberFormatException e) { // tests disable this optimisation by passing an invalid checksum