CR: Javadoc + method on store file metadata

Author: original-brownbear

server/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java

@@ -19,11 +19,13 @@
 
 package org.elasticsearch.index.store;
 
+import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -100,6 +102,29 @@ public String checksum() {
         return this.checksum;
     }
 
+    /**
+     * Checks if the bytes returned by {@link #hash()} are the contents of the file that this instances refers to.
+     *
+     * @return {@code true} iff {@link #hash()} will return the actual file contents
+     */
+    public boolean hashEqualsContents() {
+        if (hash.length == length) {
+            try {
+                final boolean checksumsMatch = Store.digestToString(CodecUtil.retrieveChecksum(
+                    new ByteArrayIndexInput("store_file", hash.bytes, hash.offset, hash.length))).equals(checksum);
+                assert checksumsMatch : "Checksums did not match for [" + this + "] which has a hash of [" + hash + "]";
+                return checksumsMatch;
+            } catch (Exception e) {
+                // Hash didn't contain any bytes that Lucene could extract a checksum from so we can't verify against the checksum of the
+                // original file. We should never see an exception here because lucene files are assumed to always contain the checksum
+                // footer.
+                assert false : new AssertionError("Saw exception for hash [" + hash + "] but expected it to be Lucene file", e);
+                return false;
+            }
+        }
+        return false;
+    }
+
     /**
      * Returns <code>true</code> iff the length and the checksums are the same. otherwise <code>false</code>
      */

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -182,6 +182,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String UPLOADED_DATA_BLOB_PREFIX = "__";
 
+    /**
+     * Prefix used for the identifiers of data blobs that were not actually written to the repository physically because their contents are
+     * already stored in the metadata referencing them, i.e. in {@link BlobStoreIndexShardSnapshot} and
+     * {@link BlobStoreIndexShardSnapshots}. This is the case for files for which {@link StoreFileMetaData#hashEqualsContents()} is
+     * {@code true}.
+     */
     private static final String VIRTUAL_DATA_BLOB_PREFIX = "v__";
 
     /**
@@ -1521,7 +1527,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s
 
                 // We can skip writing blobs where the metadata hash is equal to the blob's contents because we store the hash/contents
                 // directly in the shard level metadata in this case
-                final boolean needsWrite = md.hash().length != md.length();
+                final boolean needsWrite = md.hashEqualsContents() == false;
                 indexTotalFileCount += md.length();
                 indexTotalNumberOfFiles++;