refactored BagVerifier to use other more modular classes

Author: johnscancella

src/main/java/gov/loc/repository/bagit/exceptions/InvalidBagitFileFormatException.java

@@ -1,7 +1,7 @@
 package gov.loc.repository.bagit.exceptions;
 
 /**
- * Class to represent an error when the bag manifest file does not conform to the bagit specfication format
+ * Class to represent an error when a specific bag file does not conform to its bagit specfication format
  */
 public class InvalidBagitFileFormatException extends Exception {
   private static final long serialVersionUID = 1L;

src/main/java/gov/loc/repository/bagit/util/PathUtils.java

@@ -5,7 +5,11 @@
 import java.nio.file.Path;
 import java.nio.file.attribute.DosFileAttributes;
 
+import gov.loc.repository.bagit.domain.Bag;
+import gov.loc.repository.bagit.domain.Version;
+
 public final class PathUtils {
+  private static final String PAYLOAD_DIR_NAME = "data";
 
   private PathUtils(){
     //intentionally left blank
@@ -62,4 +66,23 @@ public static boolean isHidden(final Path path) throws IOException{
 
     return Files.isHidden(path);
   }
+  
+  /*
+   * Get the directory that contains the payload files.
+   */
+  /**
+   * With bagit version 2.0 (.bagit)
+   * payload files are no longer in the "data" directory. This method accounts for this
+   * and will return the directory that contains the payload files
+   * 
+   * @param bag that contains the payload files you want
+   * @return the directory that contains the payload files 
+   */
+  public static Path getDataDir(final Bag bag){
+    if(bag.getVersion().compareTo(new Version(2, 0)) >= 0){ //is it a .bagit version?
+      return bag.getRootDir();
+    }
+    
+    return bag.getRootDir().resolve(PAYLOAD_DIR_NAME);
+  }
 }

src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java

@@ -1,18 +1,12 @@
 package gov.loc.repository.bagit.verify;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.AbstractMap.SimpleImmutableEntry;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map.Entry;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -21,9 +15,7 @@
 import org.slf4j.LoggerFactory;
 
 import gov.loc.repository.bagit.domain.Bag;
-import gov.loc.repository.bagit.domain.FetchItem;
 import gov.loc.repository.bagit.domain.Manifest;
-import gov.loc.repository.bagit.domain.Version;
 import gov.loc.repository.bagit.exceptions.CorruptChecksumException;
 import gov.loc.repository.bagit.exceptions.FileNotInPayloadDirectoryException;
 import gov.loc.repository.bagit.exceptions.InvalidBagitFileFormatException;
@@ -37,41 +29,30 @@
 import gov.loc.repository.bagit.exceptions.VerificationException;
 import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping;
 import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping;
-import gov.loc.repository.bagit.reader.ManifestReader;
-import gov.loc.repository.bagit.util.PathUtils;
 
 /**
  * Responsible for verifying if a bag is valid, complete
  */
-@SuppressWarnings({"PMD.GodClass"}) //TODO refactor
 public final class BagVerifier {
   private static final Logger logger = LoggerFactory.getLogger(BagVerifier.class);
 
-  private static final String PAYLOAD_DIR_NAME = "data";
-  //@Incubating
-  private static final String DOT_BAGIT_DIR_NAME = ".bagit";
-  private static final String PAYLOAD_OXUM_REGEX = "\\d+\\.\\d+";
-  
-  private final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping;
+  private final PayloadVerifier manifestVerifier;
   private final ExecutorService executor;
 
   public BagVerifier(){
-    nameMapping = new StandardBagitAlgorithmNameToSupportedAlgorithmMapping();
-    executor = Executors.newCachedThreadPool();
+    this(Executors.newCachedThreadPool(), new StandardBagitAlgorithmNameToSupportedAlgorithmMapping());
   }
 
   public BagVerifier(final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){
-    this.nameMapping = nameMapping;
-    executor = Executors.newCachedThreadPool();
+    this(Executors.newCachedThreadPool(), nameMapping);
   }
 
   public BagVerifier(final ExecutorService executor){
-    nameMapping = new StandardBagitAlgorithmNameToSupportedAlgorithmMapping();
-    this.executor = executor;
+    this(executor, new StandardBagitAlgorithmNameToSupportedAlgorithmMapping());
   }
 
   public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSupportedAlgorithmMapping nameMapping){
-    this.nameMapping = nameMapping;
+    manifestVerifier = new PayloadVerifier(nameMapping);
     this.executor = executor;
   }
 
@@ -82,21 +63,7 @@ public BagVerifier(final ExecutorService executor, final BagitAlgorithmNameToSup
    * @return true if the bag can be quickly verified
    */
   public boolean canQuickVerify(final Bag bag){
-    final String payloadOxum = getPayloadOxum(bag);
-    logger.debug("Found payload-oxum [{}] for bag [{}]", payloadOxum, bag.getRootDir());
-    return payloadOxum != null && payloadOxum.matches(PAYLOAD_OXUM_REGEX) && bag.getItemsToFetch().size() == 0;
-  }
-  
-  /*
-   * Get the Payload-Oxum value from the key value pairs
-   */
-  private String getPayloadOxum(final Bag bag){
-    for(final SimpleImmutableEntry<String,String> keyValue : bag.getMetadata()){
-      if("Payload-Oxum".equals(keyValue.getKey())){
-        return keyValue.getValue();
-      }
-    }
-    return null;
+    return QuickVerifier.canQuickVerify(bag);
   }
 
   /**
@@ -112,28 +79,7 @@ private String getPayloadOxum(final Bag bag){
    * To check, run {@link BagVerifier#canQuickVerify}
    */
   public void quicklyVerify(final Bag bag, final boolean ignoreHiddenFiles) throws IOException, InvalidPayloadOxumException{
-    final String payloadOxum = getPayloadOxum(bag);
-    if(payloadOxum == null || !payloadOxum.matches(PAYLOAD_OXUM_REGEX)){
-      throw new PayloadOxumDoesNotExistException("Payload-Oxum does not exist in bag.");
-    }
-
-    final String[] parts = payloadOxum.split("\\.");
-    logger.debug("Parsing [{}] for the total byte size of the payload oxum", parts[0]);
-    final long totalSize = Long.parseLong(parts[0]);
-    logger.debug("Parsing [{}] for the number of files to find in the payload directory", parts[1]);
-    final long numberOfFiles = Long.parseLong(parts[1]);
-    
-    final Path payloadDir = getDataDir(bag);
-    final FileCountAndTotalSizeVistor vistor = new FileCountAndTotalSizeVistor(ignoreHiddenFiles);
-    Files.walkFileTree(payloadDir, vistor);
-    logger.info("supplied payload-oxum: [{}], Calculated payload-oxum: [{}.{}], for payload directory [{}]", payloadOxum, vistor.getTotalSize(), vistor.getCount(), payloadDir);
-    
-    if(totalSize != vistor.getTotalSize()){
-      throw new InvalidPayloadOxumException("Invalid total size. Expected " + totalSize + "but calculated " + vistor.getTotalSize());
-    }
-    if(numberOfFiles != vistor.getCount()){
-      throw new InvalidPayloadOxumException("Invalid file count. Expected " + numberOfFiles + "but found " + vistor.getCount() + " files");
-    }
+    QuickVerifier.quicklyVerify(bag, ignoreHiddenFiles);
   }
 
   /**
@@ -226,177 +172,22 @@ public void isComplete(final Bag bag, final boolean ignoreHiddenFiles) throws
     FileNotInPayloadDirectoryException, InterruptedException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{
     logger.info("Checking if the bag with root directory [{}] is complete.", bag.getRootDir());
 
-    final Path dataDir = getDataDir(bag);
-    
-    checkFetchItemsExist(bag.getItemsToFetch(), bag.getRootDir());
-    
-    checkBagitFileExists(bag.getRootDir(), bag.getVersion());
-    
-    checkPayloadDirectoryExists(bag);
-    
-    checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion());
-    
-    final Set<Path> allFilesListedInManifests = getAllFilesListedInManifests(bag);
-    checkAllFilesListedInManifestExist(allFilesListedInManifests);
-    
-    if(new Version(1,0).compareTo(bag.getVersion()) < 0){
-      checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(allFilesListedInManifests, dataDir, ignoreHiddenFiles);
-    }
-    else{
-      CheckAllFilesInPayloadDirAreListedInAllManifests(bag.getPayLoadManifests(), dataDir, ignoreHiddenFiles);
-    }
-  }
-  
-  /*
-   * Get the directory that contains the payload files.
-   */
-  private Path getDataDir(final Bag bag){
-    if(bag.getVersion().compareTo(new Version(2, 0)) >= 0){ //is it a .bagit version?
-      return bag.getRootDir();
-    }
-    
-    return bag.getRootDir().resolve(PAYLOAD_DIR_NAME);
-  }
-  
-  /*
-   * make sure all the fetch items exist in the data directory
-   */
-  private void checkFetchItemsExist(final List<FetchItem> items, final Path bagDir) throws FileNotInPayloadDirectoryException{
-    logger.info("Checking if all [{}] items in fetch.txt exist in the [{}]", items.size(), bagDir);
-    for(final FetchItem item : items){
-      final Path file = bagDir.resolve(item.path);
-      if(!Files.exists(file)){
-        throw new FileNotInPayloadDirectoryException("Fetch item " + item + " has not been fetched!");
-      }
-    }
-  }
-  
-  /*
-   * make sure the bagit.txt file exists
-   */
-  private void checkBagitFileExists(final Path rootDir, final Version version) throws MissingBagitFileException{
-    logger.info("Checking if bagit.txt file exists");
-    Path bagitFile = rootDir.resolve("bagit.txt");
-    //@Incubating
-    if(version.compareTo(new Version(2, 0)) >= 0){ //is it a .bagit version?
-      bagitFile = rootDir.resolve(DOT_BAGIT_DIR_NAME + File.separator + "bagit.txt");
-    }
-    
-    if(!Files.exists(bagitFile)){
-      throw new MissingBagitFileException("File [" + bagitFile + "] should exist but it doesn't");
-    }
-  }
-  
-  /*
-   * Make sure the data directory exists
-   */
-  private void checkPayloadDirectoryExists(final Bag bag) throws MissingPayloadDirectoryException{
-    logger.info("Checking if special payload directory exists (only for version 0.97 and earlier)");
-    final Path dataDir = getDataDir(bag);
-    
-    if(!Files.exists(dataDir)){
-      throw new MissingPayloadDirectoryException("File [" + dataDir + "] should exist but it doesn't");
-    }
-  }
-  
-  /*
-   * Must have at least one manifest-<ALGORITHM>.txt file
-   */
-  private void checkIfAtLeastOnePayloadManifestsExist(final Path rootDir, final Version version) throws MissingPayloadManifestException, IOException{
-    logger.info("Checking if there is at least one payload manifest in [{}]", rootDir);
-    boolean hasAtLeastOneManifest = false;
+    MandatoryVerifier.checkFetchItemsExist(bag.getItemsToFetch(), bag.getRootDir());
 
-    DirectoryStream<Path> directoryStream = Files.newDirectoryStream(rootDir);
-    //@Incubating
-    if(version.compareTo(new Version(2, 00)) >= 0){ //is it a .bagit version?
-      directoryStream = Files.newDirectoryStream(rootDir.resolve(DOT_BAGIT_DIR_NAME));
-    }
+    MandatoryVerifier.checkBagitFileExists(bag.getRootDir(), bag.getVersion());
 
-    for(final Path path : directoryStream){
-      if(PathUtils.getFilename(path).startsWith("manifest-")){
-        logger.debug("Found payload manifest file [{}]", path.getFileName());
-        hasAtLeastOneManifest = true;
-      }
-    }
+    MandatoryVerifier.checkPayloadDirectoryExists(bag);
 
-    if(!hasAtLeastOneManifest){
-      throw new MissingPayloadManifestException("Bag does not contain any payload manifest files");
-    }
+    MandatoryVerifier.checkIfAtLeastOnePayloadManifestsExist(bag.getRootDir(), bag.getVersion());
 
+    manifestVerifier.verifyPayload(bag, ignoreHiddenFiles);
   }
 
-  /*
-   * get all the files listed in all the manifests
-   */
-  private Set<Path> getAllFilesListedInManifests(final Bag bag) throws IOException, MaliciousPathException, UnsupportedAlgorithmException, InvalidBagitFileFormatException{
-    logger.debug("Getting all files listed in the manifest(s)");
-    final Set<Path> filesListedInManifests = new HashSet<>();
-    
-    DirectoryStream<Path> directoryStream = Files.newDirectoryStream(bag.getRootDir());
-    //@Incubating
-    if(bag.getVersion().compareTo(new Version(2, 00)) >= 0){ //is it a .bagit version?
-      directoryStream = Files.newDirectoryStream(bag.getRootDir().resolve(DOT_BAGIT_DIR_NAME));
-    }
-    
-    for(final Path path : directoryStream){
-      final String filename = PathUtils.getFilename(path);
-      if(filename.startsWith("tagmanifest-") || filename.startsWith("manifest-")){
-        logger.debug("Getting files and checksums listed in [{}]", path);
-        final Manifest manifest = ManifestReader.readManifest(nameMapping, path, bag.getRootDir(), bag.getFileEncoding());
-        filesListedInManifests.addAll(manifest.getFileToChecksumMap().keySet());
-      }
-    }
-    
-    return filesListedInManifests;
-  }
-  
-  /*
-   * Make sure all the listed files actually exist
-   */
-  @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
-  private void checkAllFilesListedInManifestExist(final Set<Path> files) throws FileNotInPayloadDirectoryException, InterruptedException{
-    final ExecutorService executor = Executors.newCachedThreadPool();
-    final CountDownLatch latch = new CountDownLatch(files.size());
-    final List<Path> missingFiles = new ArrayList<>();
-    
-    logger.debug("Checking if all files listed in the manifest(s) exist");
-    for(final Path file : files){
-      executor.execute(new CheckIfFileExistsTask(file, missingFiles, latch));
-    }
-    
-    latch.await();
-    executor.shutdown();
-    
-    if(!missingFiles.isEmpty()){
-      throw new FileNotInPayloadDirectoryException("Manifest(s) contains file(s) " + missingFiles + " but they don't exist!");
-    }
-  }
-  
-  /*
-   * Make sure all files in the directory are in at least 1 manifest
-   */
-  private void checkAllFilesInPayloadDirAreListedInAtLeastOneAManifest(final Set<Path> filesListedInManifests, final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException{
-    logger.debug("Checking if all payload files (files in {} dir) are listed in at least one manifest", payloadDir);
-    if(Files.exists(payloadDir)){
-      Files.walkFileTree(payloadDir, new PayloadFileExistsInAtLeastOneManifestVistor(filesListedInManifests, ignoreHiddenFiles));
-    }
-  }
-  
-  /*
-   * as per the bagit-spec 1.0+ all files have to be listed in all manifests
-   */
-  private void CheckAllFilesInPayloadDirAreListedInAllManifests(final Set<Manifest> payLoadManifests, final Path payloadDir, final boolean ignoreHiddenFiles) throws IOException{
-    logger.debug("Checking if all payload files (files in {} dir) are listed in all manifests", payloadDir);
-    if(Files.exists(payloadDir)){
-      Files.walkFileTree(payloadDir, new PayloadFileExistsInAllManifestsVistor(payLoadManifests, ignoreHiddenFiles));
-    }
-  }
-
-  public BagitAlgorithmNameToSupportedAlgorithmMapping getNameMapping() {
-    return nameMapping;
-  }
-
   public ExecutorService getExecutor() {
     return executor;
   }
+
+  public PayloadVerifier getManifestVerifier() {
+    return manifestVerifier;
+  }
 }