diff --git a/src/includes/startCLI.sh b/src/includes/startCLI.sh index 4b67034..ccfb4e4 100755 --- a/src/includes/startCLI.sh +++ b/src/includes/startCLI.sh @@ -1,19 +1,36 @@ #!/bin/bash echo "Welcome to Image Tools version: ${project.version}" -command="" +args="" +while (( "$#" )); do + args="$args $1" + shift +done +command="-Xmx1500m -cp ${project.name}-${project.version}-jfx.jar:lib/* com.sothr.imagetools.AppCLI" correct=false -while true; do - read -p "Please enter and commandline arguments you would like to include: " args - command="-Xmx1500m -cp ${project.name}-${project.version}-jfx.jar:lib/* com.sothr.imagetools.AppCLI $args" - echo "Is \"$command\" accurate? (yes/no)" +#Check for existing commands and use them instead of asking if possible +if [[ -z "$args" ]] +then + while true; do + read -p "Please enter and commandline arguments you would like to include: " args + command="$command $args" + echo "Is \"$command\" accurate? (yes/no)" + select yn in "Yes" "No"; do + case $yn in + Yes ) correct=true; java $command; exit;; + No ) break;; + esac + done + case $correct in + true ) break;; + esac + done +else + command="$command $args" + echo "Is \"$command\" accurate? (yes/no)" select yn in "Yes" "No"; do case $yn in - Yes ) correct=true; break;; - No ) break;; + Yes ) java $command; exit;; + No ) exit;; esac done - case $correct in - true ) break;; - esac -done -java $command +fi \ No newline at end of file diff --git a/src/main/java/com/sothr/imagetools/AppCLI.java b/src/main/java/com/sothr/imagetools/AppCLI.java index 3d67d8d..0a20ad9 100644 --- a/src/main/java/com/sothr/imagetools/AppCLI.java +++ b/src/main/java/com/sothr/imagetools/AppCLI.java @@ -33,8 +33,11 @@ class AppCLI { } private static Options getOptions() { + //scan a list of directories Options options = new Options(); options.addOption(new Option("s", true, "scan directories for a list of similar images")); + //scan directories in a recursive manner + options.addOption(new Option("r", false, "scan directories recursively")); return options; } @@ -42,10 +45,15 @@ class AppCLI { //scan a comma separated list of paths to search for image similarities Engine engine = new ConcurrentEngine(); if (cmd.hasOption('s')) { + Boolean recursive = false; + Integer recursiveDepth = 500; + if (cmd.hasOption('r')) { + recursive = true; + } String scanList = cmd.getOptionValue('s'); String[] paths = scanList.split(","); for (String path : paths) { - List similarImages = engine.getSimilarImagesForDirectory(path); + List similarImages = engine.getSimilarImagesForDirectory(path, recursive, recursiveDepth); for (int index = 0; index < similarImages.length(); index++) { SimilarImages similar = similarImages.apply(index); System.out.println(similar.toString()); diff --git a/src/main/scala/com/sothr/imagetools/ConcurrentEngine.scala b/src/main/scala/com/sothr/imagetools/ConcurrentEngine.scala index 11fcec4..000a01a 100644 --- a/src/main/scala/com/sothr/imagetools/ConcurrentEngine.scala +++ b/src/main/scala/com/sothr/imagetools/ConcurrentEngine.scala @@ -20,50 +20,43 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging { val engineSimilarityController = system.actorOf(Props[ConcurrentEngineSimilarityController], name = "EngineSimilarityController") implicit val timeout = Timeout(30, TimeUnit.SECONDS) - def getImagesForDirectory(directoryPath:String):List[Image] = { + def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image] = { debug(s"Looking for images in directory: $directoryPath") - val directory:File = new File(directoryPath) + val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth) val images:mutable.MutableList[Image] = new mutable.MutableList[Image]() - if (directory.isDirectory) { - val files = directory.listFiles(imageFilter) - info(s"Found ${files.length} files that are images in directory: $directoryPath") - for (file <- files) { - engineProcessingController ! EngineProcessFile(file) - } - engineProcessingController ! EngineNoMoreFiles - var doneProcessing = false - while(!doneProcessing) { - val f = engineProcessingController ? EngineIsProcessingFinished - val result = Await.result(f, timeout.duration).asInstanceOf[Boolean] - result match { - case true => - doneProcessing = true - debug("Processing Complete") - case false => - debug("Still Processing") - //sleep thread - Thread.sleep(5000L) - //val future = Future { blocking(Thread.sleep(5000L)); "done" } - } + for (file <- imageFiles) { + engineProcessingController ! EngineProcessFile(file) + } + engineProcessingController ! EngineNoMoreFiles + var doneProcessing = false + while(!doneProcessing) { + val f = engineProcessingController ? EngineIsProcessingFinished + val result = Await.result(f, timeout.duration).asInstanceOf[Boolean] + result match { + case true => + doneProcessing = true + debug("Processing Complete") + case false => + debug("Still Processing") + //sleep thread + Thread.sleep(5000L) + //val future = Future { blocking(Thread.sleep(5000L)); "done" } } - val f = engineProcessingController ? EngineGetProcessingResults - val result = Await.result(f, timeout.duration).asInstanceOf[List[Image]] - images ++= result - } else { - error(s"Provided path: $directoryPath is not a directory") } + val f = engineProcessingController ? EngineGetProcessingResults + val result = Await.result(f, timeout.duration).asInstanceOf[List[Image]] + images ++= result images.toList } //needs to be rebuilt - def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages] = { + def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages] = { debug(s"Looking for similar images in directory: $directoryPath") - val images = getImagesForDirectory(directoryPath) - engineSimilarityController ! EngineCompareSetImages(images) + val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth) info(s"Searching ${images.length} images for similarities") for (rootImage <- images) { debug(s"Looking for images similar to: ${rootImage.imagePath}") - engineSimilarityController ! EngineCompareImages(rootImage) + engineSimilarityController ! EngineCompareImages(rootImage, images) } //tell the comparison engine there's nothing left to compare engineSimilarityController ! EngineNoMoreComparisons @@ -90,7 +83,7 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging { val ignoreSet = new mutable.HashSet[Image]() for (similarImages <- result) { count += 1 - if (count % 25 == 0 || count == result.length) debug(s"Cleaning similar image $count/$result.length ${result.length-count} left to clean") + if (count % 25 == 0 || count == result.length) debug(s"Cleaning similar image $count/${result.length} ${result.length - count} left to clean") if (!ignoreSet.contains(similarImages.rootImage)) { cleanedSimilarImages += similarImages ignoreSet += similarImages.rootImage @@ -124,7 +117,6 @@ case object EngineActorProcessingFinished case object EngineActorReactivate class ConcurrentEngineProcessingController extends Actor with ActorLogging { - val imageCache = AppConfig.cacheManager.getCache("images") val numOfRouters = { val max = PropertiesService.get(PropertiesEnum.ConcurrentProcessingLimit.toString).toInt val processors = Runtime.getRuntime.availableProcessors() @@ -158,12 +150,7 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging { def processFile(command:EngineProcessFile) = { log.debug(s"Started evaluating ${command.file.getAbsolutePath}") toProcess += 1 - if (imageCache.isKeyInCache(command.file.getAbsolutePath)) { - log.debug(s"${command.file.getAbsolutePath} was already processed") - self ! EngineFileProcessed(imageCache.get(command.file.getAbsolutePath).getObjectValue.asInstanceOf[Image]) - } else { - router ! command - } + router ! command } def fileProcessed(command:EngineFileProcessed) = { @@ -246,9 +233,8 @@ class ConcurrentEngineProcessingActor extends Actor with ActorLogging { } //finding similarities between images -case class EngineCompareImages(image1:Image) +case class EngineCompareImages(image1:Image, images:List[Image]) case class EngineCompareImagesComplete(similarImages:SimilarImages) -case class EngineCompareSetImages(images:List[Image]) case object EngineNoMoreComparisons case object EngineIsSimilarityFinished case object EngineGetSimilarityResults @@ -265,7 +251,6 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { val router = context.actorOf(Props[ConcurrentEngineSimilarityActor].withRouter(RoundRobinRouter(nrOfInstances = numOfRouters))) val allSimilarImages = new mutable.MutableList[SimilarImages] - var numImages = 0 var toProcess = 0 var processed = 0 @@ -279,7 +264,6 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { override def receive = { case command:EngineCompareImages => findSimilarities(command) case command:EngineCompareImagesComplete => similarityProcessed(command) - case command:EngineCompareSetImages => setImageList(command) case EngineNoMoreComparisons => requestWrapup() case EngineActorCompareImagesFinished => actorProcessingFinished() case EngineIsSimilarityFinished => isProcessingFinished() @@ -287,24 +271,20 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { case _ => log.info("received unknown message") } - def setImageList(command:EngineCompareSetImages) = { - numImages = command.images.length - router ! Broadcast(command) - } def findSimilarities(command:EngineCompareImages) = { - log.debug(s"Finding similarities between ${command.image1.imagePath} and $numImages images") + log.debug(s"Finding similarities between {} and {} images", command.image1.imagePath, command.images.length) toProcess += 1 - if (toProcess % 250 == 0 || toProcess == numImages) { - log.info("Sent {}/{} images to be processed for similarites", toProcess, numImages) + if (toProcess % 250 == 0) { + log.info("Sent {} images to be processed for similarites", toProcess) } //just relay the command to our workers - router ! EngineCompareImages(command.image1) + router ! command } def similarityProcessed(command:EngineCompareImagesComplete) = { processed += 1 - if (processed % 25 == 0 || processed == numImages) log.info(s"Processed $processed/$toProcess") + if (processed % 25 == 0 || processed == toProcess) log.info(s"Processed $processed/$toProcess") if (command.similarImages != null) { log.debug(s"Found similar images: ${command.similarImages}") allSimilarImages += command.similarImages @@ -357,27 +337,17 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { class ConcurrentEngineSimilarityActor extends Actor with ActorLogging { var ignoreMessages = false - var imageList = new mutable.MutableList[Image]() override def receive = { case command:EngineCompareImages => compareImages(command) - case command:EngineCompareSetImages => cloneAndSetImages(command) case EngineNoMoreComparisons => finishedComparisons() case EngineActorReactivate => ignoreMessages = false case _ => log.info("received unknown message") } - def cloneAndSetImages(command:EngineCompareSetImages) = { - imageList.clear() - for (image <- command.images) { - imageList += image.cloneImage - } - log.debug("Added {} cloned images to internal list", imageList.length) - } - def compareImages(command:EngineCompareImages) = { if (!ignoreMessages) { val similarImages = new mutable.MutableList[Image]() - for (image <- imageList) { + for (image <- command.images) { if (!command.image1.equals(image)) { if (HashService.areImageHashesSimilar(command.image1.hashes, image.hashes)) { similarImages += image @@ -398,9 +368,7 @@ class ConcurrentEngineSimilarityActor extends Actor with ActorLogging { def finishedComparisons() = { if (!ignoreMessages) { - log.info("Commanded to finish processing") ignoreMessages = true - imageList.clear() log.debug("Finished processing comparisons") sender ! EngineActorCompareImagesFinished } diff --git a/src/main/scala/com/sothr/imagetools/Engine.scala b/src/main/scala/com/sothr/imagetools/Engine.scala index 6edff72..ca71457 100644 --- a/src/main/scala/com/sothr/imagetools/Engine.scala +++ b/src/main/scala/com/sothr/imagetools/Engine.scala @@ -1,6 +1,9 @@ package com.sothr.imagetools import com.sothr.imagetools.image.{SimilarImages, ImageFilter, Image} +import com.sothr.imagetools.util.DirectoryFilter +import scala.collection.mutable +import java.io.File import grizzled.slf4j.Logging /** @@ -11,13 +14,32 @@ abstract class Engine extends Logging{ val imageFilter:ImageFilter = new ImageFilter() val imageCache = AppConfig.cacheManager.getCache("images") + def getAllImageFiles(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[File] = { + val fileList = new mutable.MutableList[File]() + val directory:File = new File(directoryPath) + val imageFilter = new ImageFilter + if (directory.isDirectory) { + val files = directory.listFiles(imageFilter) + fileList ++= files + info(s"Found ${files.length} files that are images in directory: $directoryPath") + if (recursive) { + val directoryFilter = new DirectoryFilter + val directories = directory.listFiles(directoryFilter) + for (directory <- directories) { + fileList ++= getAllImageFiles(directory.getAbsolutePath, recursive, recursiveDepth-1) + } + } + } + fileList.toList + } + /** * Get all images for a directory with hashes */ - def getImagesForDirectory(directoryPath:String):List[Image]; + def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image]; /** * Get all similar images for a directory with hashes */ - def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages]; + def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages]; } diff --git a/src/main/scala/com/sothr/imagetools/SequentialEngine.scala b/src/main/scala/com/sothr/imagetools/SequentialEngine.scala index 091c16a..1ffb84f 100644 --- a/src/main/scala/com/sothr/imagetools/SequentialEngine.scala +++ b/src/main/scala/com/sothr/imagetools/SequentialEngine.scala @@ -4,43 +4,32 @@ import com.sothr.imagetools.image.{SimilarImages, ImageFilter, Image} import scala.collection.mutable import java.io.File import grizzled.slf4j.Logging -import net.sf.ehcache.Element /** * Created by drew on 1/26/14. */ class SequentialEngine extends Engine with Logging { - def getImagesForDirectory(directoryPath:String):List[Image] = { + def getImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[Image] = { debug(s"Looking for images in directory: $directoryPath") val images:mutable.MutableList[Image] = new mutable.MutableList[Image]() + val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth) val directory:File = new File(directoryPath) var count = 0 - if (directory.isDirectory) { - val files = directory.listFiles(imageFilter) - info(s"Found ${files.length} files that are images in directory: $directoryPath") - for (file <- files) { - count += 1 - if (count % 25 == 0) info(s"Processed ${count}/${files.size}") - if (imageCache.isKeyInCache(file.getAbsolutePath)) { - images += imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image] - } else { - val image = ImageService.getImage(file) - if (image != null) { - imageCache.put(new Element(file.getAbsolutePath, image)) - images += image - } - } + for (file <- imageFiles) { + count += 1 + if (count % 25 == 0) info(s"Processed ${count}/${imageFiles.size}") + val image = ImageService.getImage(file) + if (image != null) { + images += image } - } else { - error(s"Provided path: $directoryPath is not a directory") } images.toList } - def getSimilarImagesForDirectory(directoryPath:String):List[SimilarImages] = { + def getSimilarImagesForDirectory(directoryPath:String, recursive:Boolean=false, recursiveDepth:Int=500):List[SimilarImages] = { debug(s"Looking for similar images in directory: $directoryPath") - val images = getImagesForDirectory(directoryPath) + val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth) info(s"Searching ${images.length} images for similarities") val ignoreSet = new mutable.HashSet[Image]() val allSimilarImages = new mutable.MutableList[SimilarImages]() diff --git a/src/main/scala/com/sothr/imagetools/hash/HashService.scala b/src/main/scala/com/sothr/imagetools/hash/HashService.scala index d528551..880e372 100644 --- a/src/main/scala/com/sothr/imagetools/hash/HashService.scala +++ b/src/main/scala/com/sothr/imagetools/hash/HashService.scala @@ -18,7 +18,7 @@ import com.sothr.imagetools.image.Image object HashService extends Logging { def getImageHashes(imagePath:String):ImageHashDTO = { - debug(s"Creating hashes for $imagePath") + //debug(s"Creating hashes for $imagePath") getImageHashes(ImageIO.read(new File(imagePath)), imagePath) } diff --git a/src/main/scala/com/sothr/imagetools/image/Image.scala b/src/main/scala/com/sothr/imagetools/image/Image.scala index 3c1e3fb..2a0d94d 100644 --- a/src/main/scala/com/sothr/imagetools/image/Image.scala +++ b/src/main/scala/com/sothr/imagetools/image/Image.scala @@ -9,7 +9,7 @@ class Image(val imagePath:String, val thumbnailPath:String, val imageSize:Tuple2 var imageType:ImageType = ImageType.SingleFrameImage def isSimilarTo(otherImage:Image):Boolean = { - debug("Checking {} for similarities with {}",imagePath, otherImage.imagePath) + //debug(s"Checking $imagePath for similarities with ${otherImage.imagePath}") HashService.areImageHashesSimilar(this.hashes,otherImage.hashes) } diff --git a/src/main/scala/com/sothr/imagetools/image/ImageService.scala b/src/main/scala/com/sothr/imagetools/image/ImageService.scala index 561eb3f..a574360 100644 --- a/src/main/scala/com/sothr/imagetools/image/ImageService.scala +++ b/src/main/scala/com/sothr/imagetools/image/ImageService.scala @@ -8,18 +8,27 @@ import com.sothr.imagetools.image.Image import com.sothr.imagetools.hash.HashService import javax.imageio.ImageIO import java.io.IOException +import net.sf.ehcache.Element object ImageService extends Logging { + val imageCache = AppConfig.cacheManager.getCache("images") + def getImage(file:File):Image = { try { - val thumbnailPath = getThumbnailPath(file) - val bufferedImage = ImageIO.read(file) - val hashes = HashService.getImageHashes(bufferedImage, file.getAbsolutePath) - val imageSize = { (bufferedImage.getWidth, bufferedImage.getHeight) } - val image = new Image(file.getAbsolutePath, thumbnailPath, imageSize, hashes) - debug(s"Created image: $image") - return image + if (imageCache.isKeyInCache(file.getAbsolutePath)) { + debug(s"${file.getAbsolutePath} was already processed") + return imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image] + } else { + val bufferedImage = ImageIO.read(file) + val thumbnailPath = getThumbnailPath(bufferedImage, file) + val hashes = HashService.getImageHashes(bufferedImage, file.getAbsolutePath) + val imageSize = { (bufferedImage.getWidth, bufferedImage.getHeight) } + val image = new Image(file.getAbsolutePath, thumbnailPath, imageSize, hashes) + debug(s"Created image: $image") + imageCache.put(new Element(file.getAbsolutePath, image)) + return image + } } catch { case ioe:IOException => error(s"Error processing ${file.getAbsolutePath}", ioe) case ex:Exception => error(s"Error processing ${file.getAbsolutePath}", ex) @@ -27,7 +36,7 @@ object ImageService extends Logging { null } - def getThumbnailPath(file:File):String = { + def getThumbnailPath(image:BufferedImage, file:File):String = { "." } diff --git a/src/main/scala/com/sothr/imagetools/util/DirectoryFilter.scala b/src/main/scala/com/sothr/imagetools/util/DirectoryFilter.scala new file mode 100644 index 0000000..20f6941 --- /dev/null +++ b/src/main/scala/com/sothr/imagetools/util/DirectoryFilter.scala @@ -0,0 +1,13 @@ +package com.sothr.imagetools.util + +import java.io.{File, FilenameFilter} + +/** + * Created by drew on 1/26/14. + */ +class DirectoryFilter extends FilenameFilter { + + def accept(dir: File, name: String): Boolean = { + return new File(dir, name).isDirectory(); + } +}