Added a recursive option. Added some information and changed the api of the Engine class.

src/includes/startCLI.sh

@@ -1,19 +1,36 @@
 #!/bin/bash
 echo "Welcome to Image Tools version: ${project.version}"
-command=""
+args=""
+while (( "$#" )); do 
+  args="$args $1" 
+  shift 
+done
+command="-Xmx1500m -cp ${project.name}-${project.version}-jfx.jar:lib/* com.sothr.imagetools.AppCLI"
 correct=false
-while true; do
-	read -p "Please enter and commandline arguments you would like to include: " args
-	command="-Xmx1500m -cp ${project.name}-${project.version}-jfx.jar:lib/* com.sothr.imagetools.AppCLI $args"
-	echo "Is \"$command\" accurate? (yes/no)"
+#Check for existing commands and use them instead of asking if possible
+if [[ -z "$args" ]]
+then
+    while true; do
+    	read -p "Please enter and commandline arguments you would like to include: " args
+    	command="$command $args"
+    	echo "Is \"$command\" accurate? (yes/no)"
+    	select yn in "Yes" "No"; do
+    		case $yn in
+    			Yes ) correct=true; java $command; exit;;
+    			No ) break;;
+    		esac
+    	done
+    	case $correct in
+    		true ) break;;
+    	esac
+    done   
+else
+    command="$command $args"
+    echo "Is \"$command\" accurate? (yes/no)"
 	select yn in "Yes" "No"; do
 		case $yn in
-			Yes ) correct=true; break;;
-			No ) break;;
+			Yes ) java $command; exit;;
+			No ) exit;;
 		esac
 	done
-	case $correct in
-		true ) break;;
-	esac
-done	
-java $command
+fi

src/main/java/com/sothr/imagetools/AppCLI.java

@@ -33,8 +33,11 @@ class AppCLI {
   }
 
   private static Options getOptions() {
+    //scan a list of directories
     Options options = new Options();
     options.addOption(new Option("s", true, "scan directories for a list of similar images"));
+    //scan directories in a recursive manner
+    options.addOption(new Option("r", false, "scan directories recursively"));
     return options;
   }
 
@@ -42,10 +45,15 @@ class AppCLI {
     //scan a comma separated list of paths to search for image similarities
     Engine engine = new ConcurrentEngine();
     if (cmd.hasOption('s')) {
+      Boolean recursive = false;
+      Integer recursiveDepth = 500;
+      if (cmd.hasOption('r')) {
+          recursive = true;
+      } 
       String scanList = cmd.getOptionValue('s');
       String[] paths = scanList.split(",");
       for (String path : paths) {
-        List<SimilarImages> similarImages = engine.getSimilarImagesForDirectory(path);
+        List<SimilarImages> similarImages = engine.getSimilarImagesForDirectory(path, recursive, recursiveDepth);
         for (int index = 0; index < similarImages.length(); index++) {
           SimilarImages similar = similarImages.apply(index);
           System.out.println(similar.toString());

src/main/scala/com/sothr/imagetools/ConcurrentEngine.scala

@@ -20,50 +20,43 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging {
     val engineSimilarityController = system.actorOf(Props[ConcurrentEngineSimilarityController], name = "EngineSimilarityController")
     implicit val timeout = Timeout(30, TimeUnit.SECONDS)
     
-  def getImagesForDirectory(directoryPath:String):List[Image] = {
+  def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image] = {
     debug(s"Looking for images in directory: $directoryPath")
-    val directory:File = new File(directoryPath)
+    val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth)
     val images:mutable.MutableList[Image] = new mutable.MutableList[Image]()
-    if (directory.isDirectory) {
-      val files = directory.listFiles(imageFilter)
-      info(s"Found ${files.length} files that are images in directory: $directoryPath")
-      for (file <- files) {
-        engineProcessingController ! EngineProcessFile(file)
-      }
-      engineProcessingController ! EngineNoMoreFiles
-      var doneProcessing = false
-      while(!doneProcessing) {
-          val f = engineProcessingController ? EngineIsProcessingFinished
-          val result = Await.result(f, timeout.duration).asInstanceOf[Boolean]
-          result match {
-            case true =>
-              doneProcessing = true
-              debug("Processing Complete")
-            case false => 
-              debug("Still Processing")
-              //sleep thread
-              Thread.sleep(5000L)
-              //val future = Future { blocking(Thread.sleep(5000L)); "done" }
-          }
+    for (file <- imageFiles) {
+      engineProcessingController ! EngineProcessFile(file)
+    }
+    engineProcessingController ! EngineNoMoreFiles
+    var doneProcessing = false
+    while(!doneProcessing) {
+      val f = engineProcessingController ? EngineIsProcessingFinished
+      val result = Await.result(f, timeout.duration).asInstanceOf[Boolean]
+      result match {
+        case true =>
+          doneProcessing = true
+          debug("Processing Complete")
+        case false => 
+          debug("Still Processing")
+          //sleep thread
+          Thread.sleep(5000L)
+          //val future = Future { blocking(Thread.sleep(5000L)); "done" }
       }
-      val f = engineProcessingController ? EngineGetProcessingResults
-      val result = Await.result(f, timeout.duration).asInstanceOf[List[Image]]
-      images ++= result
-    } else {
-      error(s"Provided path: $directoryPath is not a directory")
     }
+    val f = engineProcessingController ? EngineGetProcessingResults
+    val result = Await.result(f, timeout.duration).asInstanceOf[List[Image]]
+    images ++= result
     images.toList
   }
 
   //needs to be rebuilt
-  def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages] = {
+  def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages] = {
     debug(s"Looking for similar images in directory: $directoryPath")
-    val images = getImagesForDirectory(directoryPath)
-    engineSimilarityController ! EngineCompareSetImages(images)
+    val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth)
     info(s"Searching ${images.length} images for similarities")
     for (rootImage <- images) {
       debug(s"Looking for images similar to: ${rootImage.imagePath}")
-      engineSimilarityController ! EngineCompareImages(rootImage)
+      engineSimilarityController ! EngineCompareImages(rootImage, images)
     }
     //tell the comparison engine there's nothing left to compare
     engineSimilarityController ! EngineNoMoreComparisons
@@ -90,7 +83,7 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging {
     val ignoreSet = new mutable.HashSet[Image]()
     for (similarImages <- result) {
         count += 1
-        if (count % 25 == 0 || count == result.length) debug(s"Cleaning similar image $count/$result.length ${result.length-count} left to clean")
+        if (count % 25 == 0 || count == result.length) debug(s"Cleaning similar image $count/${result.length} ${result.length - count} left to clean")
         if (!ignoreSet.contains(similarImages.rootImage)) {
             cleanedSimilarImages += similarImages
             ignoreSet += similarImages.rootImage
@@ -124,7 +117,6 @@ case object EngineActorProcessingFinished
 case object EngineActorReactivate
 
 class ConcurrentEngineProcessingController extends Actor with ActorLogging {
-    val imageCache = AppConfig.cacheManager.getCache("images")
     val numOfRouters = {
       val max = PropertiesService.get(PropertiesEnum.ConcurrentProcessingLimit.toString).toInt
       val processors = Runtime.getRuntime.availableProcessors()
@@ -158,12 +150,7 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging {
     def processFile(command:EngineProcessFile) = {
         log.debug(s"Started evaluating ${command.file.getAbsolutePath}")
         toProcess += 1
-        if (imageCache.isKeyInCache(command.file.getAbsolutePath)) {
-            log.debug(s"${command.file.getAbsolutePath} was already processed")
-            self ! EngineFileProcessed(imageCache.get(command.file.getAbsolutePath).getObjectValue.asInstanceOf[Image])
-        } else {
-            router ! command
-        }
+        router ! command
     }
     
     def fileProcessed(command:EngineFileProcessed) = {
@@ -246,9 +233,8 @@ class ConcurrentEngineProcessingActor extends Actor with ActorLogging {
 }
 
 //finding similarities between images
-case class EngineCompareImages(image1:Image)
+case class EngineCompareImages(image1:Image, images:List[Image])
 case class EngineCompareImagesComplete(similarImages:SimilarImages)
-case class EngineCompareSetImages(images:List[Image])
 case object EngineNoMoreComparisons
 case object EngineIsSimilarityFinished
 case object EngineGetSimilarityResults
@@ -265,7 +251,6 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
   val router = context.actorOf(Props[ConcurrentEngineSimilarityActor].withRouter(RoundRobinRouter(nrOfInstances = numOfRouters)))
 
   val allSimilarImages = new mutable.MutableList[SimilarImages]
-  var numImages = 0
   var toProcess = 0
   var processed = 0
 
@@ -279,7 +264,6 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
   override def receive = {
     case command:EngineCompareImages => findSimilarities(command)
     case command:EngineCompareImagesComplete => similarityProcessed(command)
-    case command:EngineCompareSetImages => setImageList(command)
     case EngineNoMoreComparisons => requestWrapup()
     case EngineActorCompareImagesFinished => actorProcessingFinished()
     case EngineIsSimilarityFinished => isProcessingFinished()
@@ -287,24 +271,20 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
     case _ => log.info("received unknown message")
   }
 
-  def setImageList(command:EngineCompareSetImages) = {
-      numImages = command.images.length
-      router ! Broadcast(command)
-  }
 
   def findSimilarities(command:EngineCompareImages) = {
-    log.debug(s"Finding similarities between ${command.image1.imagePath} and $numImages images")
+    log.debug(s"Finding similarities between {} and {} images", command.image1.imagePath, command.images.length)
     toProcess += 1
-    if (toProcess % 250 == 0 || toProcess == numImages) {
-        log.info("Sent {}/{} images to be processed for similarites", toProcess, numImages)
+    if (toProcess % 250 == 0) {
+        log.info("Sent {} images to be processed for similarites", toProcess)
     }
     //just relay the command to our workers
-    router ! EngineCompareImages(command.image1)
+    router ! command
   }
 
   def similarityProcessed(command:EngineCompareImagesComplete) = {
     processed += 1
-    if (processed % 25 == 0 || processed == numImages) log.info(s"Processed $processed/$toProcess")
+    if (processed % 25 == 0 || processed == toProcess) log.info(s"Processed $processed/$toProcess")
     if (command.similarImages != null) {
       log.debug(s"Found similar images: ${command.similarImages}")
       allSimilarImages += command.similarImages
@@ -357,27 +337,17 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
 
 class ConcurrentEngineSimilarityActor extends Actor with ActorLogging {
   var ignoreMessages = false
-  var imageList = new mutable.MutableList[Image]()
   override def receive = {
     case command:EngineCompareImages => compareImages(command)
-    case command:EngineCompareSetImages => cloneAndSetImages(command)
     case EngineNoMoreComparisons => finishedComparisons()
     case EngineActorReactivate => ignoreMessages = false
     case _ => log.info("received unknown message")
   }
 
-  def cloneAndSetImages(command:EngineCompareSetImages) = {
-      imageList.clear()
-      for (image <- command.images) {
-          imageList += image.cloneImage
-      }
-      log.debug("Added {} cloned images to internal list", imageList.length)
-  }
-
   def compareImages(command:EngineCompareImages) = {
     if (!ignoreMessages) {
       val similarImages = new mutable.MutableList[Image]()
-      for (image <- imageList) {
+      for (image <- command.images) {
         if (!command.image1.equals(image)) {
           if (HashService.areImageHashesSimilar(command.image1.hashes, image.hashes)) {
             similarImages += image
@@ -398,9 +368,7 @@ class ConcurrentEngineSimilarityActor extends Actor with ActorLogging {
 
   def finishedComparisons() = {
     if (!ignoreMessages) {
-        log.info("Commanded to finish processing")
       ignoreMessages = true
-      imageList.clear()
       log.debug("Finished processing comparisons")
       sender ! EngineActorCompareImagesFinished
     }

src/main/scala/com/sothr/imagetools/Engine.scala

@@ -1,6 +1,9 @@
 package com.sothr.imagetools
 
 import com.sothr.imagetools.image.{SimilarImages, ImageFilter, Image}
+import com.sothr.imagetools.util.DirectoryFilter
+import scala.collection.mutable
+import java.io.File
 import grizzled.slf4j.Logging
 
 /**
@@ -11,13 +14,32 @@ abstract class Engine extends Logging{
   val imageFilter:ImageFilter = new ImageFilter()
   val imageCache = AppConfig.cacheManager.getCache("images")
 
+  def getAllImageFiles(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[File] = {
+    val fileList = new mutable.MutableList[File]()
+    val directory:File = new File(directoryPath)
+    val imageFilter = new ImageFilter
+    if (directory.isDirectory) {
+      val files = directory.listFiles(imageFilter)
+      fileList ++= files
+      info(s"Found ${files.length} files that are images in directory: $directoryPath")
+      if (recursive) {
+        val directoryFilter = new DirectoryFilter
+        val directories = directory.listFiles(directoryFilter)
+        for (directory <- directories) {
+            fileList ++= getAllImageFiles(directory.getAbsolutePath, recursive, recursiveDepth-1)
+        }
+      }
+    }
+    fileList.toList
+  }
+
   /**
    * Get all images for a directory with hashes
    */
-  def getImagesForDirectory(directoryPath:String):List[Image];
+  def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image];
   
   /**
    * Get all similar images for a directory with hashes
    */
-  def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages];
+  def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages];
 }

src/main/scala/com/sothr/imagetools/SequentialEngine.scala

@@ -4,43 +4,32 @@ import com.sothr.imagetools.image.{SimilarImages, ImageFilter, Image}
 import scala.collection.mutable
 import java.io.File
 import grizzled.slf4j.Logging
-import net.sf.ehcache.Element
 
 /**
  * Created by drew on 1/26/14.
  */
 class SequentialEngine extends Engine with Logging {
 
-  def getImagesForDirectory(directoryPath:String):List[Image] = {
+  def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image] = {
     debug(s"Looking for images in directory: $directoryPath")
     val images:mutable.MutableList[Image] = new mutable.MutableList[Image]()
+    val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth)
     val directory:File = new File(directoryPath)
     var count = 0
-    if (directory.isDirectory) {
-      val files = directory.listFiles(imageFilter)
-      info(s"Found ${files.length} files that are images in directory: $directoryPath")
-      for (file <- files) {
-        count += 1
-        if (count % 25 == 0) info(s"Processed ${count}/${files.size}")
-        if (imageCache.isKeyInCache(file.getAbsolutePath)) {
-          images += imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image]
-        } else {
-          val image = ImageService.getImage(file)
-          if (image != null) {
-            imageCache.put(new Element(file.getAbsolutePath, image))
-            images += image
-          }
-        }
+    for (file <- imageFiles) {
+      count += 1
+      if (count % 25 == 0) info(s"Processed ${count}/${imageFiles.size}")
+      val image = ImageService.getImage(file)
+      if (image != null) {
+        images += image
       }
-    } else {
-      error(s"Provided path: $directoryPath is not a directory")
     }
     images.toList
   }
 
-  def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages] = {
+  def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages] = {
     debug(s"Looking for similar images in directory: $directoryPath")
-    val images = getImagesForDirectory(directoryPath)
+    val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth)
     info(s"Searching ${images.length} images for similarities")
     val ignoreSet = new mutable.HashSet[Image]()
     val allSimilarImages = new mutable.MutableList[SimilarImages]()

src/main/scala/com/sothr/imagetools/hash/HashService.scala

@@ -18,7 +18,7 @@ import com.sothr.imagetools.image.Image
 object HashService extends Logging {
 
   def getImageHashes(imagePath:String):ImageHashDTO = {
-    debug(s"Creating hashes for $imagePath")
+    //debug(s"Creating hashes for $imagePath")
     getImageHashes(ImageIO.read(new File(imagePath)), imagePath)
   }
 

src/main/scala/com/sothr/imagetools/image/Image.scala

@@ -9,7 +9,7 @@ class Image(val imagePath:String, val thumbnailPath:String, val imageSize:Tuple2
   var imageType:ImageType = ImageType.SingleFrameImage
 
   def isSimilarTo(otherImage:Image):Boolean = {
-    debug("Checking {} for similarities with {}",imagePath, otherImage.imagePath)
+    //debug(s"Checking $imagePath for similarities with ${otherImage.imagePath}")
     HashService.areImageHashesSimilar(this.hashes,otherImage.hashes)
   }
 

src/main/scala/com/sothr/imagetools/image/ImageService.scala

@@ -8,18 +8,27 @@ import com.sothr.imagetools.image.Image
 import com.sothr.imagetools.hash.HashService
 import javax.imageio.ImageIO
 import java.io.IOException
+import net.sf.ehcache.Element
 
 object ImageService extends Logging {
 
+  val imageCache = AppConfig.cacheManager.getCache("images")
+
   def getImage(file:File):Image = {
     try {
-      val thumbnailPath = getThumbnailPath(file)
-      val bufferedImage = ImageIO.read(file)
-      val hashes = HashService.getImageHashes(bufferedImage, file.getAbsolutePath)
-      val imageSize = { (bufferedImage.getWidth, bufferedImage.getHeight) }
-      val image = new Image(file.getAbsolutePath, thumbnailPath, imageSize, hashes)
-      debug(s"Created image: $image")
-      return image
+      if (imageCache.isKeyInCache(file.getAbsolutePath)) {
+        debug(s"${file.getAbsolutePath} was already processed")
+        return imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image]
+      } else {
+        val bufferedImage = ImageIO.read(file)
+        val thumbnailPath = getThumbnailPath(bufferedImage, file)
+        val hashes = HashService.getImageHashes(bufferedImage, file.getAbsolutePath)
+        val imageSize = { (bufferedImage.getWidth, bufferedImage.getHeight) }
+        val image = new Image(file.getAbsolutePath, thumbnailPath, imageSize, hashes)
+        debug(s"Created image: $image")
+        imageCache.put(new Element(file.getAbsolutePath, image))
+        return image
+      }
     } catch {
       case ioe:IOException => error(s"Error processing ${file.getAbsolutePath}", ioe)
       case ex:Exception => error(s"Error processing ${file.getAbsolutePath}", ex)
@@ -27,7 +36,7 @@ object ImageService extends Logging {
     null
   }
 
-  def getThumbnailPath(file:File):String = {
+  def getThumbnailPath(image:BufferedImage, file:File):String = {
     "."
   }
 

src/main/scala/com/sothr/imagetools/util/DirectoryFilter.scala

@@ -0,0 +1,13 @@
+package com.sothr.imagetools.util
+
+import java.io.{File, FilenameFilter}
+
+/**
+ * Created by drew on 1/26/14.
+ */
+class DirectoryFilter extends FilenameFilter {
+
+  def accept(dir: File, name: String): Boolean = {
+    return new File(dir, name).isDirectory();
+  }
+}