Browse Source

Fix for the concurrent engine.

Made the concurrent engine the default again.
Updated the license
Added a script to clean the logs in the base directory.
Updated the .gitignore to ignore configuration files in the base directory needed for individual runs from within intellij.
Improved the handling around cached images, falling back on restoring the data from the database when the ehcache fails.
master
Drew Short 10 years ago
parent
commit
b701ea28e8
  1. 4
      .gitignore
  2. 2
      LICENSE
  3. 6
      cleanLogs.sh
  4. 17
      src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala
  5. 6
      src/main/scala/com/sothr/imagetools/image/ImageService.scala
  6. 4
      src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala

4
.gitignore

@ -38,3 +38,7 @@ version.info
#Cache directory #Cache directory
.cache/ .cache/
#Root Config Files
/user.conf
/logback.xml

2
LICENSE

@ -1,6 +1,6 @@
The MIT License (MIT) The MIT License (MIT)
Copyright (c) 2013 Drew Short
Copyright (c) 2014 Drew Short
Permission is hereby granted, free of charge, to any person obtaining a copy of Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in this software and associated documentation files (the "Software"), to deal in

6
cleanLogs.sh

@ -0,0 +1,6 @@
rm ImageTools.debug
rm ImageTools.debug.*
rm ImageTools.info
rm ImageTools.info.*
rm ImageTools.err
rm ImageTools.err.*

17
src/main/scala/com/sothr/imagetools/engine/ConcurrentEngine.scala

@ -31,6 +31,8 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging {
debug(s"Looking for images in directory: $directoryPath") debug(s"Looking for images in directory: $directoryPath")
val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth) val imageFiles = getAllImageFiles(directoryPath, recursive, recursiveDepth)
val images:mutable.MutableList[Image] = new mutable.MutableList[Image]() val images:mutable.MutableList[Image] = new mutable.MutableList[Image]()
// make sure the engine is listening
engineProcessingController ! EngineStart
for (file <- imageFiles) { for (file <- imageFiles) {
engineProcessingController ! EngineProcessFile(file) engineProcessingController ! EngineProcessFile(file)
} }
@ -61,6 +63,8 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging {
debug(s"Looking for similar images in directory: $directoryPath") debug(s"Looking for similar images in directory: $directoryPath")
val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth) val images = getImagesForDirectory(directoryPath, recursive, recursiveDepth)
info(s"Searching ${images.length} images for similarities") info(s"Searching ${images.length} images for similarities")
// make sure the engine is listening
engineSimilarityController ! EngineStart
for (rootImage <- images) { for (rootImage <- images) {
debug(s"Looking for images similar to: ${rootImage.imagePath}") debug(s"Looking for images similar to: ${rootImage.imagePath}")
engineSimilarityController ! EngineCompareImages(rootImage, images) engineSimilarityController ! EngineCompareImages(rootImage, images)
@ -113,6 +117,7 @@ class ConcurrentEngine extends Engine with grizzled.slf4j.Logging {
// external cases // // external cases //
case class SetNewListener(listenerType: ActorRef) case class SetNewListener(listenerType: ActorRef)
case object EngineStart
// processing files into images // processing files into images
case class EngineProcessFile(file:File) case class EngineProcessFile(file:File)
@ -133,7 +138,7 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging {
if (processors > max) threads = max else if (processors > 1) threads = processors - 1 else threads = 1 if (processors > max) threads = max else if (processors > 1) threads = processors - 1 else threads = 1
threads threads
} }
val router = context.actorOf(Props[ConcurrentEngineProcessingActor].withRouter(SmallestMailboxRouter(nrOfInstances = numOfRouters)))
var router = context.actorOf(Props[ConcurrentEngineProcessingActor].withRouter(SmallestMailboxRouter(nrOfInstances = numOfRouters)))
var images:mutable.MutableList[Image] = new mutable.MutableList[Image]() var images:mutable.MutableList[Image] = new mutable.MutableList[Image]()
var toProcess = 0 var toProcess = 0
@ -159,6 +164,7 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging {
case command:SetNewListener => setListener(command.listenerType) case command:SetNewListener => setListener(command.listenerType)
case command:EngineProcessFile => processFile(command) case command:EngineProcessFile => processFile(command)
case command:EngineFileProcessed => fileProcessed(command) case command:EngineFileProcessed => fileProcessed(command)
case EngineStart => startEngine()
case EngineNoMoreFiles => requestWrapup() case EngineNoMoreFiles => requestWrapup()
case EngineActorProcessingFinished => actorProcessingFinished() case EngineActorProcessingFinished => actorProcessingFinished()
case EngineIsProcessingFinished => checkIfProcessingIsFinished() case EngineIsProcessingFinished => checkIfProcessingIsFinished()
@ -171,6 +177,10 @@ class ConcurrentEngineProcessingController extends Actor with ActorLogging {
this.listener ! PoisonPill this.listener ! PoisonPill
} }
def startEngine() = {
router ! Broadcast(EngineActorReactivate)
}
def processFile(command:EngineProcessFile) = { def processFile(command:EngineProcessFile) = {
log.debug(s"Started evaluating ${command.file.getAbsolutePath}") log.debug(s"Started evaluating ${command.file.getAbsolutePath}")
toProcess += 1 toProcess += 1
@ -302,6 +312,7 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
case command:SetNewListener => setListener(command.listenerType) case command:SetNewListener => setListener(command.listenerType)
case command:EngineCompareImages => findSimilarities(command) case command:EngineCompareImages => findSimilarities(command)
case command:EngineCompareImagesComplete => similarityProcessed(command) case command:EngineCompareImagesComplete => similarityProcessed(command)
case EngineStart => startEngine()
case EngineNoMoreComparisons => requestWrapup() case EngineNoMoreComparisons => requestWrapup()
case EngineActorCompareImagesFinished => actorProcessingFinished() case EngineActorCompareImagesFinished => actorProcessingFinished()
case EngineIsSimilarityFinished => checkIfProcessingIsFinished() case EngineIsSimilarityFinished => checkIfProcessingIsFinished()
@ -314,6 +325,10 @@ class ConcurrentEngineSimilarityController extends Actor with ActorLogging {
this.listener ! PoisonPill this.listener ! PoisonPill
} }
def startEngine() = {
router ! Broadcast(EngineActorReactivate)
}
def findSimilarities(command:EngineCompareImages) = { def findSimilarities(command:EngineCompareImages) = {
log.debug(s"Finding similarities between {} and {} images", command.image1.imagePath, command.images.length) log.debug(s"Finding similarities between {} and {} images", command.image1.imagePath, command.images.length)
toProcess += 1 toProcess += 1

6
src/main/scala/com/sothr/imagetools/image/ImageService.scala

@ -23,11 +23,11 @@ object ImageService extends Logging {
//get from memory cache if possible //get from memory cache if possible
try { try {
if (imageCache.isKeyInCache(file.getAbsolutePath)) { if (imageCache.isKeyInCache(file.getAbsolutePath)) {
found = true
image = imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image] image = imageCache.get(file.getAbsolutePath).getObjectValue.asInstanceOf[Image]
found = true
} }
} catch { } catch {
case npe:NullPointerException => error(s"Error grabbing \'${file.getAbsolutePath}\' from cache", npe)
case npe:NullPointerException => debug(s"\'${file.getAbsolutePath}\' was supposed to be in the cache, but was not")
} }
//get from datastore if possible //get from datastore if possible
if (!found) { if (!found) {
@ -35,7 +35,7 @@ object ImageService extends Logging {
val tempImage = imageDAO.find(file.getAbsolutePath) val tempImage = imageDAO.find(file.getAbsolutePath)
if (tempImage != null) image = tempImage if (tempImage != null) image = tempImage
} catch { } catch {
case ex:Exception => error(s"Error looking up \'${file.getAbsolutePath}\' from database", ex)
case ex:Exception => error(s"Error looking up \'${file.getAbsolutePath}\' was supposed to be in the database, but was not", ex)
} }
} }
image image

4
src/main/scala/com/sothr/imagetools/ui/controller/AppController.scala

@ -10,7 +10,7 @@ import javafx.scene.web.WebView
import javafx.scene.{Group, Node, Scene} import javafx.scene.{Group, Node, Scene}
import javafx.stage.{DirectoryChooser, Stage, StageStyle} import javafx.stage.{DirectoryChooser, Stage, StageStyle}
import com.sothr.imagetools.engine.{Engine, SequentialEngine}
import com.sothr.imagetools.engine.{ConcurrentEngine, Engine}
import com.sothr.imagetools.ui.component.ImageTileFactory import com.sothr.imagetools.ui.component.ImageTileFactory
import com.sothr.imagetools.util.{PropertiesService, ResourceLoader} import com.sothr.imagetools.util.{PropertiesService, ResourceLoader}
import grizzled.slf4j.Logging import grizzled.slf4j.Logging
@ -33,7 +33,7 @@ class AppController extends Logging {
@FXML var selectedDirectoryLabel: javafx.scene.control.Label = null @FXML var selectedDirectoryLabel: javafx.scene.control.Label = null
// Engine // Engine
val engine:Engine = new SequentialEngine()
val engine:Engine = new ConcurrentEngine()
// Current State // Current State
var currentDirectory:String = "." var currentDirectory:String = "."

Loading…
Cancel
Save