From ab5c39c667491cdb8b329ebbebf00be15b21412a Mon Sep 17 00:00:00 2001 From: Drew Short Date: Wed, 19 Nov 2014 21:45:29 -0600 Subject: [PATCH] Proper cleaning of similar images. Working on a sane sorting process for the similar images. --- .../imagetools/engine/ConcurrentEngine.scala | 11 ++-- .../com/sothr/imagetools/engine/Engine.scala | 60 ++++++++++++++----- .../imagetools/engine/SequentialEngine.scala | 3 +- .../engine/image/SimilarImages.scala | 16 +++-- .../sothr/imagetools/engine/EngineTest.scala | 4 +- .../ui/controller/AppController.scala | 3 +- 6 files changed, 67 insertions(+), 30 deletions(-) diff --git a/engine/src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala b/engine/src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala index 3ef14c4..097c066 100644 --- a/engine/src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala +++ b/engine/src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala @@ -219,7 +219,7 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging { */ def checkForResults() = { try { - listener ! SubmitMessage(s"Finished Processing $processed/$processed images") + listener ! SubmitMessage(s"Finished processing $processed/$processed images") processorsFinished = 0 toProcess = 0 processed = 0 @@ -367,7 +367,7 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { */ def checkIfProcessingIsFinished() = { try { - log.debug("Processors Finished {}/{}", processorsFinished, numOfRouters) + log.debug("Processors finished {}/{}", processorsFinished, numOfRouters) if (processorsFinished >= numOfRouters) sender ! true else sender ! false } catch { case e: Exception ⇒ @@ -381,7 +381,7 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging { */ def checkForResults() = { try { - listener ! SubmitMessage(s"Finished Scanning $processed/$processed images") + listener ! SubmitMessage(s"Finished scanning $processed/$processed images") processorsFinished = 0 toProcess = 0 processed = 0 @@ -422,8 +422,9 @@ class ConcurrentEngineSimilarityActor extends Actor with ActorLogging { } //only send a message if we find similar images if (similarImages.length >= 1) { - val similarImage = new SimilarImages(command.image1, similarImages.toList) - log.debug(s"Found ${similarImage.similarImages.length} similar images to ${similarImage.rootImage}") + similarImages += command.image1 + val similarImage = new SimilarImages(similarImages.toSet) + log.debug(s"Found ${similarImage.similarImages.size} similar images to ${command.image1}") sender ! EngineCompareImagesComplete(similarImage) } else { log.debug(s"Found no similar images to ${command.image1}") diff --git a/engine/src/main/scala/com/sothr/imagetools/engine/Engine.scala b/engine/src/main/scala/com/sothr/imagetools/engine/Engine.scala index 5afe288..c4df2e0 100644 --- a/engine/src/main/scala/com/sothr/imagetools/engine/Engine.scala +++ b/engine/src/main/scala/com/sothr/imagetools/engine/Engine.scala @@ -3,11 +3,13 @@ package com.sothr.imagetools.engine import java.io.File import akka.actor._ -import com.sothr.imagetools.engine.image.{Image, ImageFilter, ImageService, SimilarImages} +import com.sothr.imagetools.engine.image._ import com.sothr.imagetools.engine.util.DirectoryFilter import grizzled.slf4j.Logging +import scala.collection.immutable.HashSet import scala.collection.mutable +import scala.util.control.Breaks._ /** * Engine definition @@ -87,29 +89,55 @@ abstract class Engine extends Logging { */ def processSimilarities(similarList: List[SimilarImages]): List[SimilarImages] = { //process the result into a list we want in cleanedSimilarImages + /* + Go through all the images. If a similar image for the image doesn't exist, create it. if it does, + add it to that similar image. The end result is that all images should be grouped according to + their similar images and images that are similar to them. + */ var count = 0 + // similar image mapping to map known images back to their similar set + val similarImageMap = new mutable.HashMap[Image, SimilarImages]() + + // List of the actual similar image sets val cleanedSimilarImages = new mutable.MutableList[SimilarImages]() - val ignoreSet = new mutable.HashSet[Image]() + + // loop over all of the similar image sets for (similarImages <- similarList) { count += 1 - if (count % 25 == 0 || count == similarList.length) debug(s"Cleaning similar image $count/${similarList.length} ${similarList.length - count} left to clean") - if (!ignoreSet.contains(similarImages.rootImage)) { - cleanedSimilarImages += similarImages - ignoreSet += similarImages.rootImage - for (image <- similarImages.similarImages) { - ignoreSet += image + if (count % 25 == 0 || count == similarList.length) { + debug(s"Cleaning similar images. $count/${similarList.length} ${similarList.length - count} left to clean") + this.searchedListener ! SubmitMessage(s"Cleaning similar images. $count/${similarList.length}") + } + var foundSimilarity = false + var similarity:SimilarImages = null + breakable { for (similarImage <- similarImages.similarImages) { + if (similarImageMap.contains(similarImage)) { + similarity = similarImageMap(similarImage) + foundSimilarity = true + break() } + } } + + //if no similarity was found, one should be created + if (!foundSimilarity) { + similarity = new SimilarImages(new HashSet[Image]) + // the created similarity is added to the cleaned list + cleanedSimilarImages += similarity } - //rewrite todo: - /* - Go through all the images. If a similar image for the image doesn't exist, create it. if it does, - add it to that similar image. The end result is that all images should be grouped according to - their similar images and images that are similar to them. - */ + + // all images should be added to this new similarity + similarity.similarImages = similarity.similarImages ++ similarImages.similarImages + similarImages.similarImages.foreach(img => similarImageMap.put(img, similarity)) } - //return a non mutable cleaned list - cleanedSimilarImages.toList + //get a count of similar images found + var totalCount = 0 + cleanedSimilarImages.foreach(img => totalCount += img.similarImages.size) + debug(s"Found $totalCount images with similarities") + this.searchedListener ! SubmitMessage(s"Found $totalCount images with similarities") + + // Sort the similarImages by ?!?!?!? and return + cleanedSimilarImages.toList.sorted(SimilarImagesOrdering) } } diff --git a/engine/src/main/scala/com/sothr/imagetools/engine/SequentialEngine.scala b/engine/src/main/scala/com/sothr/imagetools/engine/SequentialEngine.scala index b795f44..a12daa5 100644 --- a/engine/src/main/scala/com/sothr/imagetools/engine/SequentialEngine.scala +++ b/engine/src/main/scala/com/sothr/imagetools/engine/SequentialEngine.scala @@ -58,7 +58,8 @@ class SequentialEngine extends Engine with Logging { } } if (similarImages.length > 1) { - val similar = new SimilarImages(rootImage, similarImages.toList) + similarImages += rootImage + val similar = new SimilarImages(similarImages.toSet) debug(s"Found similar images: ${similar.toString}") allSimilarImages += similar } diff --git a/engine/src/main/scala/com/sothr/imagetools/engine/image/SimilarImages.scala b/engine/src/main/scala/com/sothr/imagetools/engine/image/SimilarImages.scala index de38840..a0e0c41 100644 --- a/engine/src/main/scala/com/sothr/imagetools/engine/image/SimilarImages.scala +++ b/engine/src/main/scala/com/sothr/imagetools/engine/image/SimilarImages.scala @@ -7,11 +7,11 @@ import grizzled.slf4j.Logging * * Created by drew on 1/26/14. */ -class SimilarImages(val rootImage: Image, val similarImages: List[Image]) extends Logging { +class SimilarImages(var similarImages: Set[Image]) extends Logging { override def hashCode: Int = { val prime = 7 - var result = prime * 1 + rootImage.hashCode + var result = prime * similarImages.size for (similarImage <- similarImages) { result = prime * result + similarImage.hashCode } @@ -19,8 +19,7 @@ class SimilarImages(val rootImage: Image, val similarImages: List[Image]) extend } override def toString: String = { - s"""RootImage: ${rootImage.imagePath} - Similar Images: + s"""Similar Images: $getPrettySimilarImagesList""".stripMargin } @@ -33,4 +32,13 @@ class SimilarImages(val rootImage: Image, val similarImages: List[Image]) extend sb.toString() } + def ordering() = { + 1 * similarImages.size + similarImages.groupBy(img => img.getImagePath) + } + +} + +object SimilarImagesOrdering extends Ordering[SimilarImages] { + def compare(a:SimilarImages, b:SimilarImages) = a.ordering() compare b.ordering() } diff --git a/engine/src/test/scala/com/sothr/imagetools/engine/EngineTest.scala b/engine/src/test/scala/com/sothr/imagetools/engine/EngineTest.scala index 3c45255..8ad9f28 100644 --- a/engine/src/test/scala/com/sothr/imagetools/engine/EngineTest.scala +++ b/engine/src/test/scala/com/sothr/imagetools/engine/EngineTest.scala @@ -20,7 +20,7 @@ class EngineTest extends BaseTest { similarImages.length } assertResult(2) { - similarImages(0).similarImages.length + similarImages(0).similarImages.size } } @@ -38,7 +38,7 @@ class EngineTest extends BaseTest { similarImages.length } assertResult(2) { - similarImages(0).similarImages.length + similarImages(0).similarImages.size } } } diff --git a/gui/src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala b/gui/src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala index 0fd308c..59d03f9 100644 --- a/gui/src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala +++ b/gui/src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala @@ -333,8 +333,7 @@ class AppController extends Logging { val similarImages = engine.getSimilarImagesForDirectory(currentDirectory, recursive = doRecursiveProcessing.isSelected) val tempImages = new mutable.MutableList[Image]() for (similarImage <- similarImages) { - debug(s"Adding similar images ${similarImage.rootImage.toString} to app") - tempImages += similarImage.rootImage + debug(s"Adding similar images ${similarImage.toString} to app") similarImage.similarImages.foreach(image => tempImages += image) } tempImages.toList